@celilo/console-server 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +23 -0
- package/src/index.ts +17 -0
- package/src/serve.ts +236 -0
- package/src/upstream.ts +173 -0
- package/src/verbs.ts +379 -0
package/package.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@celilo/console-server",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "HTTP surface for the celilo web console. Serves the SPA and hosts the tsrpc API. Holds no database: it reaches celilo-mgr over the SSH remote API as a read-only principal.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./src/index.ts",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./src/index.ts"
|
|
9
|
+
},
|
|
10
|
+
"scripts": {
|
|
11
|
+
"test": "bun test --timeout 30000"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"src/",
|
|
15
|
+
"README.md"
|
|
16
|
+
],
|
|
17
|
+
"license": "MIT",
|
|
18
|
+
"dependencies": {
|
|
19
|
+
"@celilo/console-protocol": "*",
|
|
20
|
+
"@celilo/core": "*",
|
|
21
|
+
"@celilo/cli-display": "*"
|
|
22
|
+
}
|
|
23
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The console server's public surface.
|
|
3
|
+
*
|
|
4
|
+
* `serve` and `createHandler` are the two ways in: one binds a port, the other
|
|
5
|
+
* returns a handler a test can drive with a `Request`. The verbs are exported
|
|
6
|
+
* because the e2e suite asserts against their output directly rather than
|
|
7
|
+
* through HTTP, which keeps a shape assertion from also being a routing test.
|
|
8
|
+
*/
|
|
9
|
+
export { serve, createHandler, type ServeOptions, type AuthGate, type AuthSubject } from './serve';
|
|
10
|
+
export { Upstream, UpstreamUnavailable, type UnavailableReason } from './upstream';
|
|
11
|
+
export {
|
|
12
|
+
readTopology,
|
|
13
|
+
readModules,
|
|
14
|
+
readClosure,
|
|
15
|
+
readAlerts,
|
|
16
|
+
readBackupsWithCoverage,
|
|
17
|
+
} from './verbs';
|
package/src/serve.ts
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The console's HTTP surface: the tsrpc API, the SPA, and the gate in front.
|
|
3
|
+
*
|
|
4
|
+
* Holds no database handle. Every fact it serves comes from `Upstream`, which
|
|
5
|
+
* reaches celilo-mgr over the SSH remote API as a principal granted read ops
|
|
6
|
+
* only. That is a security boundary, not a deployment preference: a console
|
|
7
|
+
* sharing celilo's process would hold the whole database including the
|
|
8
|
+
* encrypted secret store, and would be constrained only by its own code.
|
|
9
|
+
*
|
|
10
|
+
* It is a Bun server rather than tsrpc's own `HttpServer` for one reason worth
|
|
11
|
+
* stating. This process serves two things on one port — an RPC surface and a
|
|
12
|
+
* single-page app — and the SPA needs a catch-all that returns `index.html` for
|
|
13
|
+
* any path the router owns. Running tsrpc's server alongside a static server
|
|
14
|
+
* means two ports, and two ports behind one VPN-only ingress is two things to
|
|
15
|
+
* get wrong. So the RPC calls are dispatched by hand onto the same handlers.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { ServiceType } from '@celilo/console-protocol';
|
|
19
|
+
import { Upstream, type UpstreamOptions, UpstreamUnavailable } from './upstream';
|
|
20
|
+
import {
|
|
21
|
+
readAlerts,
|
|
22
|
+
readBackupsWithCoverage,
|
|
23
|
+
readClosure,
|
|
24
|
+
readModules,
|
|
25
|
+
readTopology,
|
|
26
|
+
} from './verbs';
|
|
27
|
+
|
|
28
|
+
export interface ServeOptions {
|
|
29
|
+
port: number;
|
|
30
|
+
/** `celilo-api@<server>`. */
|
|
31
|
+
dest: string;
|
|
32
|
+
identityFile: string;
|
|
33
|
+
/** Directory holding the built SPA. */
|
|
34
|
+
spaDir: string;
|
|
35
|
+
/** Decides who is asking. See `AuthGate`. */
|
|
36
|
+
authenticate: AuthGate;
|
|
37
|
+
cacheTtlMs?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Injectable so tests drive the whole surface without SSH, matching
|
|
40
|
+
* `UpstreamOptions`. Absent in production, where the real SSH transport is
|
|
41
|
+
* selected by `identityFile`.
|
|
42
|
+
*/
|
|
43
|
+
openTransport?: UpstreamOptions['openTransport'];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Who is asking, decided before any handler runs.
|
|
48
|
+
*
|
|
49
|
+
* Returns the authenticated subject, or null to refuse. This is a function
|
|
50
|
+
* rather than baked-in OIDC so the e2e suite can drive the server with a stub
|
|
51
|
+
* and so the identity provider can change without touching request routing.
|
|
52
|
+
*
|
|
53
|
+
* There is no default. A server constructed without one would answer to
|
|
54
|
+
* anybody, and "we forgot to pass the gate" must not be a thing that silently
|
|
55
|
+
* works (Rule 6.4: deny by default).
|
|
56
|
+
*/
|
|
57
|
+
export type AuthGate = (request: Request) => Promise<AuthSubject | null>;
|
|
58
|
+
|
|
59
|
+
export interface AuthSubject {
|
|
60
|
+
/** Stable identifier from the identity provider. */
|
|
61
|
+
subject: string;
|
|
62
|
+
/** Display name, when the provider supplied one. */
|
|
63
|
+
name?: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Every RPC path the protocol declares, and nothing else. */
|
|
67
|
+
type ApiName = keyof ServiceType['api'];
|
|
68
|
+
|
|
69
|
+
const READ_VERBS: ApiName[] = ['Topology', 'Modules', 'Closure', 'Alerts', 'Backups'];
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Map an upstream failure onto an HTTP status and a tsrpc-shaped error.
|
|
73
|
+
*
|
|
74
|
+
* Every reason is REPORTED. None becomes an empty success. A panel that receives
|
|
75
|
+
* `{ alerts: [] }` because the read was denied shows a quiet fleet, and a quiet
|
|
76
|
+
* fleet is the one answer this console must never give by accident.
|
|
77
|
+
*/
|
|
78
|
+
function unavailableResponse(error: UpstreamUnavailable): Response {
|
|
79
|
+
const status = error.reason === 'denied' ? 403 : error.reason === 'unknown-verb' ? 501 : 502;
|
|
80
|
+
return Response.json(
|
|
81
|
+
{
|
|
82
|
+
isSucc: false,
|
|
83
|
+
err: {
|
|
84
|
+
message: error.message,
|
|
85
|
+
type: 'ApiError',
|
|
86
|
+
code: error.reason,
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
{ status },
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The MIME types the SPA build actually emits. */
|
|
94
|
+
const CONTENT_TYPES: Record<string, string> = {
|
|
95
|
+
'.html': 'text/html; charset=utf-8',
|
|
96
|
+
'.js': 'text/javascript; charset=utf-8',
|
|
97
|
+
'.css': 'text/css; charset=utf-8',
|
|
98
|
+
'.svg': 'image/svg+xml',
|
|
99
|
+
'.woff2': 'font/woff2',
|
|
100
|
+
'.json': 'application/json',
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
function contentType(path: string): string {
|
|
104
|
+
const dot = path.lastIndexOf('.');
|
|
105
|
+
return (dot >= 0 ? CONTENT_TYPES[path.slice(dot)] : undefined) ?? 'application/octet-stream';
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Dispatch one RPC call.
|
|
110
|
+
*
|
|
111
|
+
* The request body is the tsrpc request object. Only the five read verbs are
|
|
112
|
+
* routed. `AckAlert` is declared in the protocol and deliberately NOT wired
|
|
113
|
+
* here: it is a write, it needs an authenticated subject that maps to a real
|
|
114
|
+
* `people` row, and the grant to perform it does not exist yet. An unrouted
|
|
115
|
+
* verb 404s, which is honest. Wiring it to something that silently no-ops
|
|
116
|
+
* would be worse than not having it.
|
|
117
|
+
*/
|
|
118
|
+
async function callApi(
|
|
119
|
+
name: ApiName,
|
|
120
|
+
body: Record<string, unknown>,
|
|
121
|
+
upstream: Upstream,
|
|
122
|
+
): Promise<Response> {
|
|
123
|
+
switch (name) {
|
|
124
|
+
case 'Topology':
|
|
125
|
+
return Response.json({ isSucc: true, res: await readTopology(upstream) });
|
|
126
|
+
case 'Modules':
|
|
127
|
+
return Response.json({ isSucc: true, res: await readModules(upstream) });
|
|
128
|
+
case 'Closure': {
|
|
129
|
+
const moduleId = body.moduleId;
|
|
130
|
+
if (typeof moduleId !== 'string' || moduleId.length === 0) {
|
|
131
|
+
return Response.json(
|
|
132
|
+
{ isSucc: false, err: { message: 'moduleId is required', type: 'ApiError' } },
|
|
133
|
+
{ status: 400 },
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const depth = typeof body.depth === 'number' ? body.depth : undefined;
|
|
137
|
+
return Response.json({ isSucc: true, res: await readClosure(upstream, moduleId, depth) });
|
|
138
|
+
}
|
|
139
|
+
case 'Alerts': {
|
|
140
|
+
const moduleId = typeof body.moduleId === 'string' ? body.moduleId : undefined;
|
|
141
|
+
return Response.json({ isSucc: true, res: await readAlerts(upstream, moduleId) });
|
|
142
|
+
}
|
|
143
|
+
case 'Backups':
|
|
144
|
+
return Response.json({ isSucc: true, res: await readBackupsWithCoverage(upstream) });
|
|
145
|
+
default:
|
|
146
|
+
return Response.json(
|
|
147
|
+
{ isSucc: false, err: { message: `${name} is not served here`, type: 'ApiError' } },
|
|
148
|
+
{ status: 404 },
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Build the request handler. Exported separately from `serve` so tests can
|
|
155
|
+
* drive it with a `Request` and no listening socket.
|
|
156
|
+
*/
|
|
157
|
+
export function createHandler(options: ServeOptions): (request: Request) => Promise<Response> {
|
|
158
|
+
const upstream = new Upstream({
|
|
159
|
+
dest: options.dest,
|
|
160
|
+
identityFile: options.identityFile,
|
|
161
|
+
...(options.cacheTtlMs === undefined ? {} : { cacheTtlMs: options.cacheTtlMs }),
|
|
162
|
+
...(options.openTransport === undefined ? {} : { openTransport: options.openTransport }),
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
return async (request: Request): Promise<Response> => {
|
|
166
|
+
const url = new URL(request.url);
|
|
167
|
+
|
|
168
|
+
// The gate runs FIRST, for every path including the SPA. Serving the shell
|
|
169
|
+
// to an unauthenticated caller would leak the fleet's shape through the
|
|
170
|
+
// bundle even with every API call refused.
|
|
171
|
+
const subject = await options.authenticate(request);
|
|
172
|
+
if (subject === null) {
|
|
173
|
+
return new Response('Unauthorized', {
|
|
174
|
+
status: 401,
|
|
175
|
+
headers: { 'WWW-Authenticate': 'Bearer' },
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (url.pathname.startsWith('/api/')) {
|
|
180
|
+
const name = url.pathname.slice('/api/'.length) as ApiName;
|
|
181
|
+
if (!READ_VERBS.includes(name)) {
|
|
182
|
+
return Response.json(
|
|
183
|
+
{ isSucc: false, err: { message: `Unknown verb: ${name}`, type: 'ApiError' } },
|
|
184
|
+
{ status: 404 },
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
let body: Record<string, unknown> = {};
|
|
189
|
+
try {
|
|
190
|
+
body = (await request.json()) as Record<string, unknown>;
|
|
191
|
+
} catch {
|
|
192
|
+
// A verb taking no arguments is called with an empty body by some
|
|
193
|
+
// clients and `{}` by others. Both are fine; only a malformed body
|
|
194
|
+
// with content is not, and that fails on the field check below.
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
return await callApi(name, body, upstream);
|
|
199
|
+
} catch (error) {
|
|
200
|
+
if (error instanceof UpstreamUnavailable) return unavailableResponse(error);
|
|
201
|
+
throw error;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// Static, then the SPA catch-all. The order matters: a request for
|
|
206
|
+
// `/assets/index.js` must find the file, and a request for `/backups` must
|
|
207
|
+
// find `index.html`, because the route belongs to the client-side router.
|
|
208
|
+
const asset = Bun.file(`${options.spaDir}${url.pathname}`);
|
|
209
|
+
if (url.pathname !== '/' && (await asset.exists())) {
|
|
210
|
+
return new Response(asset, { headers: { 'content-type': contentType(url.pathname) } });
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const shell = Bun.file(`${options.spaDir}/index.html`);
|
|
214
|
+
if (!(await shell.exists())) {
|
|
215
|
+
// Said out loud rather than 404ing. A console server with no SPA built
|
|
216
|
+
// into it is a packaging failure, and a bare 404 sends the reader looking
|
|
217
|
+
// for a routing bug instead.
|
|
218
|
+
return new Response(
|
|
219
|
+
`No SPA found at ${options.spaDir}. The console server ships the built client inside its package, and this one was started without it.`,
|
|
220
|
+
{ status: 500 },
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
return new Response(shell, { headers: { 'content-type': CONTENT_TYPES['.html'] as string } });
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function serve(options: ServeOptions): { port: number; stop: () => void } {
|
|
228
|
+
const handler = createHandler(options);
|
|
229
|
+
const server = Bun.serve({ port: options.port, fetch: handler });
|
|
230
|
+
// `server.port` is optional in the type because a unix-socket server has
|
|
231
|
+
// none. This one always binds a port, and asserting that is better than
|
|
232
|
+
// defaulting to 0 and reporting a port nothing is listening on.
|
|
233
|
+
const bound = server.port;
|
|
234
|
+
if (bound === undefined) throw new Error('console server started without a port');
|
|
235
|
+
return { port: bound, stop: () => server.stop(true) };
|
|
236
|
+
}
|
package/src/upstream.ts
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How the console reaches celilo.
|
|
3
|
+
*
|
|
4
|
+
* The console server holds NO database handle. Every fact it serves arrives over
|
|
5
|
+
* the SSH remote API as a principal whose grants are read-only, the same way
|
|
6
|
+
* `@celilo/mcp` reaches celilo-mgr. That is a security boundary rather than a
|
|
7
|
+
* deployment preference: a console sharing celilo's process would hold the whole
|
|
8
|
+
* database including the encrypted secret store, and would be constrained only
|
|
9
|
+
* by its own code.
|
|
10
|
+
*
|
|
11
|
+
* Two policies here differ deliberately from the MCP's client.
|
|
12
|
+
*
|
|
13
|
+
* An interview is a FAILURE, not something to answer. A browser has no
|
|
14
|
+
* interview responder attached, so a command that parks on a question would hang
|
|
15
|
+
* forever. The console is specified never to invoke an operation that can raise
|
|
16
|
+
* one, and this is where that is enforced rather than assumed.
|
|
17
|
+
*
|
|
18
|
+
* An unavailable verb is reported, never blanked. The console and the management
|
|
19
|
+
* server upgrade by different paths and are expected to run at different
|
|
20
|
+
* versions, so a verb the server does not recognise or has not granted is
|
|
21
|
+
* ordinary. Returning an empty result for it would be indistinguishable from
|
|
22
|
+
* "nothing is wrong", and only one of those means the fleet is fine.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import type { DisplayWriter } from '@celilo/cli-display';
|
|
26
|
+
import { type RemoteTransport, runRemoteClient, sshTransportWith } from '@celilo/core';
|
|
27
|
+
|
|
28
|
+
/** Why a read could not be served. Every case is renderable; none is silence. */
|
|
29
|
+
export type UnavailableReason =
|
|
30
|
+
| 'denied' // the principal is not granted this op
|
|
31
|
+
| 'unknown-verb' // an older management server does not have it
|
|
32
|
+
| 'unreachable' // ssh or the server itself did not answer
|
|
33
|
+
| 'interview' // the command parked on a question nothing here can answer
|
|
34
|
+
| 'malformed'; // it answered, but not with the JSON we asked for
|
|
35
|
+
|
|
36
|
+
export class UpstreamUnavailable extends Error {
|
|
37
|
+
constructor(
|
|
38
|
+
readonly reason: UnavailableReason,
|
|
39
|
+
readonly argv: readonly string[],
|
|
40
|
+
message: string,
|
|
41
|
+
) {
|
|
42
|
+
super(message);
|
|
43
|
+
this.name = 'UpstreamUnavailable';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface UpstreamOptions {
|
|
48
|
+
/** `celilo-api@<server>`. */
|
|
49
|
+
dest: string;
|
|
50
|
+
/** Private key selecting the console's read-only principal. */
|
|
51
|
+
identityFile: string;
|
|
52
|
+
/** Injectable so tests exercise the round trip without SSH. */
|
|
53
|
+
openTransport?: (dest: string) => RemoteTransport;
|
|
54
|
+
/** How long a cached read stays fresh, in ms. */
|
|
55
|
+
cacheTtlMs?: number;
|
|
56
|
+
/** Injected so tests control time rather than sleeping. */
|
|
57
|
+
now?: () => number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Exit code the API uses for "this principal may not do that". */
|
|
61
|
+
const EXIT_DENIED = 126;
|
|
62
|
+
|
|
63
|
+
interface CacheEntry {
|
|
64
|
+
at: number;
|
|
65
|
+
value: unknown;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export class Upstream {
|
|
69
|
+
private readonly cache = new Map<string, CacheEntry>();
|
|
70
|
+
private readonly ttl: number;
|
|
71
|
+
private readonly now: () => number;
|
|
72
|
+
|
|
73
|
+
constructor(private readonly options: UpstreamOptions) {
|
|
74
|
+
this.ttl = options.cacheTtlMs ?? 2_000;
|
|
75
|
+
this.now = options.now ?? Date.now;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Run one read verb and parse its JSON.
|
|
80
|
+
*
|
|
81
|
+
* Cached by argv, so a poll that changes nothing costs one round trip rather
|
|
82
|
+
* than one per browser tab. The TTL is short because this is a liveness
|
|
83
|
+
* display: stale-by-seconds is fine, stale-by-minutes is a lie.
|
|
84
|
+
*/
|
|
85
|
+
async read<T>(argv: readonly string[]): Promise<T> {
|
|
86
|
+
const key = argv.join(' ');
|
|
87
|
+
const hit = this.cache.get(key);
|
|
88
|
+
if (hit && this.now() - hit.at < this.ttl) return hit.value as T;
|
|
89
|
+
|
|
90
|
+
const value = await this.runJson<T>(argv);
|
|
91
|
+
this.cache.set(key, { at: this.now(), value });
|
|
92
|
+
return value;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Drop everything cached. Used when the operator asks for a hard refresh. */
|
|
96
|
+
invalidate(): void {
|
|
97
|
+
this.cache.clear();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private async runJson<T>(argv: readonly string[]): Promise<T> {
|
|
101
|
+
const lines: string[] = [];
|
|
102
|
+
const out: DisplayWriter = {
|
|
103
|
+
write: (s: string) => {
|
|
104
|
+
lines.push(s);
|
|
105
|
+
},
|
|
106
|
+
// No ANSI animation to strip back out of the payload.
|
|
107
|
+
isTTY: false,
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
let outcome: Awaited<ReturnType<typeof runRemoteClient>>;
|
|
111
|
+
try {
|
|
112
|
+
outcome = await runRemoteClient(this.options.dest, [...argv], {
|
|
113
|
+
openTransport: this.options.openTransport ?? sshTransportWith(this.options.identityFile),
|
|
114
|
+
out,
|
|
115
|
+
// Reached only if celilo asks the console a question. Nothing here can
|
|
116
|
+
// answer one, so say so loudly rather than parking a request forever.
|
|
117
|
+
renderInterview: () => {
|
|
118
|
+
throw new UpstreamUnavailable(
|
|
119
|
+
'interview',
|
|
120
|
+
argv,
|
|
121
|
+
`\`${argv.join(' ')}\` raised an interview question. The console has no responder and never invokes operations that can ask one.`,
|
|
122
|
+
);
|
|
123
|
+
},
|
|
124
|
+
onBlocked: 'return',
|
|
125
|
+
});
|
|
126
|
+
} catch (error) {
|
|
127
|
+
if (error instanceof UpstreamUnavailable) throw error;
|
|
128
|
+
throw new UpstreamUnavailable(
|
|
129
|
+
'unreachable',
|
|
130
|
+
argv,
|
|
131
|
+
`Could not reach ${this.options.dest}: ${error instanceof Error ? error.message : String(error)}`,
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (outcome.status === 'blocked') {
|
|
136
|
+
throw new UpstreamUnavailable(
|
|
137
|
+
'interview',
|
|
138
|
+
argv,
|
|
139
|
+
`\`${argv.join(' ')}\` parked on a question the console cannot answer.`,
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const output = lines.join('').trim();
|
|
144
|
+
if (outcome.exitCode !== 0) {
|
|
145
|
+
throw new UpstreamUnavailable(classifyFailure(outcome.exitCode, output), argv, output);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
return JSON.parse(output) as T;
|
|
150
|
+
} catch {
|
|
151
|
+
throw new UpstreamUnavailable(
|
|
152
|
+
'malformed',
|
|
153
|
+
argv,
|
|
154
|
+
`\`${argv.join(' ')}\` succeeded but did not return JSON.`,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Tell "you may not" from "I do not have that" from "something else broke".
|
|
162
|
+
*
|
|
163
|
+
* These render differently and an operator needs to tell them apart: a denial is
|
|
164
|
+
* a grant to fix, an unknown verb is a management server to upgrade, and
|
|
165
|
+
* anything else is a fault.
|
|
166
|
+
*/
|
|
167
|
+
function classifyFailure(exitCode: number, output: string): UnavailableReason {
|
|
168
|
+
if (exitCode === EXIT_DENIED || /permission denied|not granted/i.test(output)) return 'denied';
|
|
169
|
+
if (/unknown command|subcommand required|not a celilo command/i.test(output)) {
|
|
170
|
+
return 'unknown-verb';
|
|
171
|
+
}
|
|
172
|
+
return 'unreachable';
|
|
173
|
+
}
|
package/src/verbs.ts
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The six read verbs, as pure mappings from celilo's CLI JSON to the protocol.
|
|
3
|
+
*
|
|
4
|
+
* Pure on purpose. Each one takes an `Upstream` and returns a response, and
|
|
5
|
+
* every judgement in the file is about a shape rather than about HTTP, so the
|
|
6
|
+
* whole surface is testable with a fake upstream and no server, no socket and
|
|
7
|
+
* no SSH.
|
|
8
|
+
*
|
|
9
|
+
* Two rules run through all of them.
|
|
10
|
+
*
|
|
11
|
+
* A failed read THROWS. It never returns an empty list. `UpstreamUnavailable`
|
|
12
|
+
* carries why (denied, unknown verb, unreachable, interview, malformed), the
|
|
13
|
+
* tsrpc layer turns it into an error the client renders as "unavailable", and
|
|
14
|
+
* the console draws that differently from "nothing is wrong". Returning `[]` for
|
|
15
|
+
* a read that did not happen is the bug this console was specified around.
|
|
16
|
+
*
|
|
17
|
+
* Nothing here writes. The console's API principal is granted read ops only, so
|
|
18
|
+
* a mutating verb added to this file would be refused at the API boundary rather
|
|
19
|
+
* than running — but it would still be a lie in the source, so it stays absent.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import type {
|
|
23
|
+
AlertProjection,
|
|
24
|
+
BackupProjection,
|
|
25
|
+
BackupRunProjection,
|
|
26
|
+
ClosureNode,
|
|
27
|
+
ModuleProjection,
|
|
28
|
+
ResAlerts,
|
|
29
|
+
ResBackups,
|
|
30
|
+
ResClosure,
|
|
31
|
+
ResModules,
|
|
32
|
+
ResTopology,
|
|
33
|
+
} from '@celilo/console-protocol';
|
|
34
|
+
import type { Upstream } from './upstream';
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* How far back the backups read asks for.
|
|
38
|
+
*
|
|
39
|
+
* The grid draws days, and days past this are drawn as "not captured" rather
|
|
40
|
+
* than as missed backups. Thirty is a month of context at a payload the poll can
|
|
41
|
+
* afford: a module failing hourly produces ~720 rows in that window, which is
|
|
42
|
+
* large but bounded, where an unbounded read of the live fleet is 586 rows today
|
|
43
|
+
* and grows forever.
|
|
44
|
+
*/
|
|
45
|
+
const BACKUP_WINDOW_DAYS = 30;
|
|
46
|
+
|
|
47
|
+
/** `celilo console status --json`, verbatim. */
|
|
48
|
+
interface CliConsoleStatus {
|
|
49
|
+
zones: string[];
|
|
50
|
+
modules: {
|
|
51
|
+
id: string;
|
|
52
|
+
version: string;
|
|
53
|
+
state: string;
|
|
54
|
+
health: { cell: string; monitored: boolean; firingCount: number; suppressed: boolean };
|
|
55
|
+
systems: { hostname: string; address: string; zone: string }[];
|
|
56
|
+
lastBackupAt: number | null;
|
|
57
|
+
lastBackupFailed: boolean;
|
|
58
|
+
unwatched: boolean;
|
|
59
|
+
pagesNobody: boolean;
|
|
60
|
+
backupStale: boolean;
|
|
61
|
+
}[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* `celilo console get <id> --json`, verbatim.
|
|
66
|
+
*
|
|
67
|
+
* Note what is NOT here: delegation chains. D6 establishes that the firewall
|
|
68
|
+
* chain is derivable (a provider's capability data names the zone it delegates
|
|
69
|
+
* to) and that the derivation must be verified against a live
|
|
70
|
+
* `celilo capability info firewall` before anything is built on it. That
|
|
71
|
+
* verification has not happened and `computeClosure` does not produce chains,
|
|
72
|
+
* so this server has none to serve.
|
|
73
|
+
*/
|
|
74
|
+
interface CliClosure {
|
|
75
|
+
nodes: { moduleId: string; hop: number; optional: boolean; via: string[] }[];
|
|
76
|
+
depth: number;
|
|
77
|
+
truncated: boolean;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** `celilo alerts --json`, verbatim. */
|
|
81
|
+
interface CliAlert {
|
|
82
|
+
id: string;
|
|
83
|
+
key: string;
|
|
84
|
+
state: string;
|
|
85
|
+
severity: string;
|
|
86
|
+
monitor: string | null;
|
|
87
|
+
escalationPolicy: string | null;
|
|
88
|
+
escalationStep: number;
|
|
89
|
+
nextEscalationAt: number | null;
|
|
90
|
+
firstFiredAt: number;
|
|
91
|
+
lastSeenAt: number;
|
|
92
|
+
suppressedByAlertId: string | null;
|
|
93
|
+
suppressedByWindowId: string | null;
|
|
94
|
+
awaitingConfirmation: boolean;
|
|
95
|
+
ackedBy: string | null;
|
|
96
|
+
ackedAt: number | null;
|
|
97
|
+
silencedUntil: number | null;
|
|
98
|
+
message: string;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** `celilo backup list --json`, verbatim. */
|
|
102
|
+
interface CliBackups {
|
|
103
|
+
asOf: number;
|
|
104
|
+
windowDays: number | null;
|
|
105
|
+
backups: {
|
|
106
|
+
id: string;
|
|
107
|
+
shortId: string;
|
|
108
|
+
moduleId: string | null;
|
|
109
|
+
backupType: string;
|
|
110
|
+
status: string;
|
|
111
|
+
sizeBytes: number | null;
|
|
112
|
+
startedAt: number;
|
|
113
|
+
completedAt: number | null;
|
|
114
|
+
storage: string;
|
|
115
|
+
storagePath: string;
|
|
116
|
+
moduleVersion: string | null;
|
|
117
|
+
schemaVersion: string | null;
|
|
118
|
+
name: string | null;
|
|
119
|
+
error: string | null;
|
|
120
|
+
}[];
|
|
121
|
+
modules: {
|
|
122
|
+
moduleId: string;
|
|
123
|
+
lastSuccessAt: number | null;
|
|
124
|
+
lastAttemptAt: number | null;
|
|
125
|
+
consecutiveFailures: number;
|
|
126
|
+
}[];
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The health cell, narrowed to the four the protocol names.
|
|
131
|
+
*
|
|
132
|
+
* `not deployed` is a real cell celilo emits for a module it has installed but
|
|
133
|
+
* placed nowhere, and it maps to `not observed` rather than to `ok`: there is
|
|
134
|
+
* nothing running to be healthy. Anything unrecognised also becomes
|
|
135
|
+
* `not observed`, never `ok`, because an unknown state is not evidence that a
|
|
136
|
+
* module is fine and a newer celilo may name one this console has not seen.
|
|
137
|
+
*/
|
|
138
|
+
function healthCell(cell: string): ModuleProjection['health']['cell'] {
|
|
139
|
+
if (cell === 'firing') return 'firing';
|
|
140
|
+
if (cell === 'suppressed') return 'suppressed';
|
|
141
|
+
if (cell === 'ok') return 'ok';
|
|
142
|
+
return 'not observed';
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function toModule(row: CliConsoleStatus['modules'][number]): ModuleProjection {
|
|
146
|
+
return {
|
|
147
|
+
id: row.id,
|
|
148
|
+
version: row.version,
|
|
149
|
+
state: row.state,
|
|
150
|
+
health: { cell: healthCell(row.health.cell), firingCount: row.health.firingCount },
|
|
151
|
+
systems: row.systems.map((system) => ({
|
|
152
|
+
hostname: system.hostname,
|
|
153
|
+
address: system.address,
|
|
154
|
+
zone: system.zone,
|
|
155
|
+
})),
|
|
156
|
+
lastBackupAt: row.lastBackupAt,
|
|
157
|
+
backupStale: row.backupStale,
|
|
158
|
+
unwatched: row.unwatched,
|
|
159
|
+
pagesNobody: row.pagesNobody,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export async function readTopology(upstream: Upstream): Promise<ResTopology> {
|
|
164
|
+
const status = await upstream.read<CliConsoleStatus>(['console', 'status', '--json']);
|
|
165
|
+
return { zones: status.zones, modules: status.modules.map(toModule) };
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** The same read as the topology. The roster and the map are one poll (D9). */
|
|
169
|
+
export async function readModules(upstream: Upstream): Promise<ResModules> {
|
|
170
|
+
const status = await upstream.read<CliConsoleStatus>(['console', 'status', '--json']);
|
|
171
|
+
return { modules: status.modules.map(toModule) };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* A module's bounded closure, joined against the roster.
|
|
176
|
+
*
|
|
177
|
+
* celilo's closure names modules by id; the protocol carries the whole
|
|
178
|
+
* projection on each node, so the console can draw a dependency without a second
|
|
179
|
+
* lookup per node. The join happens here rather than in the browser because both
|
|
180
|
+
* reads are cached upstream and doing it there would mean shipping the roster
|
|
181
|
+
* twice.
|
|
182
|
+
*
|
|
183
|
+
* A node naming a module the roster does not have is DROPPED, not faked. That
|
|
184
|
+
* only happens if the two reads straddle an uninstall, and inventing a
|
|
185
|
+
* projection for it would draw a module that no longer exists.
|
|
186
|
+
*/
|
|
187
|
+
export async function readClosure(
|
|
188
|
+
upstream: Upstream,
|
|
189
|
+
moduleId: string,
|
|
190
|
+
depth?: number,
|
|
191
|
+
): Promise<ResClosure> {
|
|
192
|
+
const argv = ['console', 'get', moduleId, '--json'];
|
|
193
|
+
if (depth !== undefined) argv.push('--depth', String(depth));
|
|
194
|
+
|
|
195
|
+
const [closure, status] = await Promise.all([
|
|
196
|
+
upstream.read<CliClosure>(argv),
|
|
197
|
+
upstream.read<CliConsoleStatus>(['console', 'status', '--json']),
|
|
198
|
+
]);
|
|
199
|
+
|
|
200
|
+
const byId = new Map(status.modules.map((row) => [row.id, toModule(row)]));
|
|
201
|
+
const nodes: ClosureNode[] = [];
|
|
202
|
+
for (const node of closure.nodes) {
|
|
203
|
+
const module = byId.get(node.moduleId);
|
|
204
|
+
if (!module) continue;
|
|
205
|
+
nodes.push({ module, hop: node.hop, optional: node.optional, via: node.via });
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
nodes,
|
|
210
|
+
// EMPTY BECAUSE THERE IS NO PRODUCER, not because this fleet has no
|
|
211
|
+
// delegation. On the live fleet `iptables` delegates upstream to `axon`, so
|
|
212
|
+
// an empty list here is wrong rather than merely unpopulated. It is left
|
|
213
|
+
// empty rather than faked, and no console surface renders it today — the
|
|
214
|
+
// moment one does, it must say "not computed" rather than "none".
|
|
215
|
+
chains: [],
|
|
216
|
+
depth: closure.depth,
|
|
217
|
+
truncated: closure.truncated,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** Severity, narrowed. An unknown one is `critical`: never quieter than told. */
|
|
222
|
+
function severity(value: string): AlertProjection['severity'] {
|
|
223
|
+
return value === 'warning' ? 'warning' : 'critical';
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Alert state, narrowed. An unknown one is `firing`, for the same reason. */
|
|
227
|
+
function alertState(value: string): AlertProjection['state'] {
|
|
228
|
+
const known = ['pending', 'firing', 'acked', 'suppressed', 'resolved'] as const;
|
|
229
|
+
return (known as readonly string[]).includes(value)
|
|
230
|
+
? (value as AlertProjection['state'])
|
|
231
|
+
: 'firing';
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* The module an alert is about, from its key.
|
|
236
|
+
*
|
|
237
|
+
* Keys are structured `module:caddy/check:cert-validity`, so the subject is the
|
|
238
|
+
* first segment's value. A key that names something other than a module (a
|
|
239
|
+
* machine, a builtin check) yields null rather than a guess, and the console
|
|
240
|
+
* renders those against the fleet rather than against a module.
|
|
241
|
+
*/
|
|
242
|
+
function moduleOfKey(key: string): string | null {
|
|
243
|
+
const [head] = key.split('/');
|
|
244
|
+
if (!head?.startsWith('module:')) return null;
|
|
245
|
+
return head.slice('module:'.length) || null;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export async function readAlerts(upstream: Upstream, moduleId?: string): Promise<ResAlerts> {
|
|
249
|
+
// `alerts list`, not bare `alerts`. The API authorises `command:subcommand`,
|
|
250
|
+
// and the read-only grant this console holds is `alerts:list` — a bare
|
|
251
|
+
// `alerts` carries no subcommand token and is refused, which would have
|
|
252
|
+
// rendered as an unexplained 403 on the first real deploy.
|
|
253
|
+
const rows = await upstream.read<CliAlert[]>(['alerts', 'list', '--json']);
|
|
254
|
+
const asOf = Date.now();
|
|
255
|
+
|
|
256
|
+
// Filtered HERE rather than by asking celilo for a filtered list. The whole
|
|
257
|
+
// fleet's alerts are one cached read shared by every browser tab, and the
|
|
258
|
+
// dashboard's dock re-scopes on every selection change.
|
|
259
|
+
const wanted = moduleId ? rows.filter((row) => moduleOfKey(row.key) === moduleId) : rows;
|
|
260
|
+
|
|
261
|
+
return {
|
|
262
|
+
asOf,
|
|
263
|
+
alerts: wanted.map(
|
|
264
|
+
(row): AlertProjection => ({
|
|
265
|
+
id: row.id,
|
|
266
|
+
key: row.key,
|
|
267
|
+
moduleId: moduleOfKey(row.key),
|
|
268
|
+
message: row.message,
|
|
269
|
+
state: alertState(row.state),
|
|
270
|
+
severity: severity(row.severity),
|
|
271
|
+
policy: row.escalationPolicy,
|
|
272
|
+
firstFiredAt: row.firstFiredAt,
|
|
273
|
+
lastSeenAt: row.lastSeenAt,
|
|
274
|
+
suppressedByAlertId: row.suppressedByAlertId,
|
|
275
|
+
suppressedByWindowId: row.suppressedByWindowId,
|
|
276
|
+
silencedUntil: row.silencedUntil,
|
|
277
|
+
// `alerts --json` resolves the person to a name. There is no id on the
|
|
278
|
+
// wire, so the name serves as both — which is what the console shows.
|
|
279
|
+
ackedBy: row.ackedBy ? { id: row.ackedBy, name: row.ackedBy } : null,
|
|
280
|
+
ackedAt: row.ackedAt,
|
|
281
|
+
escalationStep: row.escalationStep,
|
|
282
|
+
nextEscalationAt: row.nextEscalationAt,
|
|
283
|
+
}),
|
|
284
|
+
),
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export async function readBackups(upstream: Upstream): Promise<ResBackups> {
|
|
289
|
+
const data = await upstream.read<CliBackups>([
|
|
290
|
+
'backup',
|
|
291
|
+
'list',
|
|
292
|
+
'--json',
|
|
293
|
+
'--since',
|
|
294
|
+
String(BACKUP_WINDOW_DAYS),
|
|
295
|
+
]);
|
|
296
|
+
|
|
297
|
+
// Every module celilo reports a backup ATTEMPT for is hooked, by definition:
|
|
298
|
+
// nothing else can produce one. Modules with no attempts are not listed here
|
|
299
|
+
// at all, and the console pairs this against the roster to find them.
|
|
300
|
+
const backups: BackupProjection[] = data.modules.map((row) => ({
|
|
301
|
+
moduleId: row.moduleId,
|
|
302
|
+
coverage: 'hooked',
|
|
303
|
+
lastSuccessAt: row.lastSuccessAt,
|
|
304
|
+
lastAttemptAt: row.lastAttemptAt,
|
|
305
|
+
lastAttemptFailed:
|
|
306
|
+
row.lastAttemptAt !== null &&
|
|
307
|
+
(row.lastSuccessAt === null || row.lastAttemptAt > row.lastSuccessAt),
|
|
308
|
+
failuresSinceSuccess: row.consecutiveFailures,
|
|
309
|
+
// celilo resolves the cadence and reports staleness on the roster read, so
|
|
310
|
+
// it is not re-derived here. Null says "this read does not carry it".
|
|
311
|
+
scheduleHours: null,
|
|
312
|
+
stale: false,
|
|
313
|
+
}));
|
|
314
|
+
|
|
315
|
+
const runs: BackupRunProjection[] = data.backups
|
|
316
|
+
// A system-state backup belongs to no module and has no row in the grid.
|
|
317
|
+
.filter((row) => row.moduleId !== null)
|
|
318
|
+
.map((row) => ({
|
|
319
|
+
id: row.shortId,
|
|
320
|
+
moduleId: row.moduleId as string,
|
|
321
|
+
at: row.startedAt,
|
|
322
|
+
completed: row.status === 'completed',
|
|
323
|
+
sizeBytes: row.sizeBytes,
|
|
324
|
+
storage: row.storage,
|
|
325
|
+
moduleVersion: row.moduleVersion,
|
|
326
|
+
name: row.name,
|
|
327
|
+
error: row.error,
|
|
328
|
+
}));
|
|
329
|
+
|
|
330
|
+
return {
|
|
331
|
+
asOf: data.asOf,
|
|
332
|
+
windowDays: data.windowDays ?? BACKUP_WINDOW_DAYS,
|
|
333
|
+
backups,
|
|
334
|
+
runs,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* Which module ids the roster knows about, paired with the backup summary.
|
|
340
|
+
*
|
|
341
|
+
* Exported because the coverage answer needs BOTH reads and neither alone is
|
|
342
|
+
* enough: `backup list` cannot name a module that has never attempted a backup,
|
|
343
|
+
* and the roster cannot say whether a module declares the hook. A module in the
|
|
344
|
+
* roster with no attempts is `unhooked` here, which is right for the eighteen
|
|
345
|
+
* modules on the live fleet that declare no `on_backup` (celilo#1131) and wrong
|
|
346
|
+
* only for a hooked module whose very first backup has not run yet. That case
|
|
347
|
+
* corrects itself on the first attempt, and the alternative — reporting eighteen
|
|
348
|
+
* modules as overdue — does not.
|
|
349
|
+
*/
|
|
350
|
+
export async function readBackupsWithCoverage(upstream: Upstream): Promise<ResBackups> {
|
|
351
|
+
const [backups, status] = await Promise.all([
|
|
352
|
+
readBackups(upstream),
|
|
353
|
+
upstream.read<CliConsoleStatus>(['console', 'status', '--json']),
|
|
354
|
+
]);
|
|
355
|
+
|
|
356
|
+
const attempted = new Set(backups.backups.map((row) => row.moduleId));
|
|
357
|
+
const staleByModule = new Map(status.modules.map((row) => [row.id, row.backupStale]));
|
|
358
|
+
|
|
359
|
+
const withStaleness = backups.backups.map((row) => ({
|
|
360
|
+
...row,
|
|
361
|
+
stale: staleByModule.get(row.moduleId) ?? false,
|
|
362
|
+
}));
|
|
363
|
+
|
|
364
|
+
const unhooked: BackupProjection[] = status.modules
|
|
365
|
+
.filter((row) => !attempted.has(row.id))
|
|
366
|
+
.map((row) => ({
|
|
367
|
+
moduleId: row.id,
|
|
368
|
+
coverage: 'unhooked' as const,
|
|
369
|
+
lastSuccessAt: null,
|
|
370
|
+
lastAttemptAt: null,
|
|
371
|
+
lastAttemptFailed: false,
|
|
372
|
+
failuresSinceSuccess: 0,
|
|
373
|
+
scheduleHours: null,
|
|
374
|
+
// Not overdue. There is nothing to run.
|
|
375
|
+
stale: false,
|
|
376
|
+
}));
|
|
377
|
+
|
|
378
|
+
return { ...backups, backups: [...withStaleness, ...unhooked] };
|
|
379
|
+
}
|