@celilo/cli 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +4 -2
- package/drizzle/0020_dns_registrations_drop_ip.sql +25 -0
- package/drizzle/0021_dns_registration_consumers.sql +63 -0
- package/drizzle/0022_dns_registrations_companion.sql +15 -0
- package/drizzle/0023_public_dns_evidence.sql +19 -0
- package/drizzle/meta/_journal.json +29 -1
- package/package.json +4 -4
- package/schemas/system_config.json +22 -11
- package/src/api/remote-client.test.ts +34 -12
- package/src/api/serve.ts +234 -38
- package/src/api/sessions.test.ts +196 -0
- package/src/api/sessions.ts +278 -0
- package/src/cli/commands/backup-sweep.ts +25 -9
- package/src/cli/commands/dns.ts +8 -4
- package/src/cli/commands/events.ts +64 -5
- package/src/cli/commands/module-update.ts +34 -11
- package/src/cli/commands/system-audit.ts +15 -0
- package/src/cli/commands/system-migrate.test.ts +25 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +1 -0
- package/src/cli/index.ts +22 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/dns-registrations-migration.test.ts +205 -0
- package/src/db/schema.ts +77 -8
- package/src/hooks/define-hook.test.ts +3 -3
- package/src/hooks/executor.test.ts +58 -0
- package/src/hooks/executor.ts +67 -7
- package/src/hooks/run-named-hook.ts +7 -1
- package/src/hooks/test-fixtures/silent-hook.ts +20 -0
- package/src/module/packaging/build.ts +14 -0
- package/src/services/alerting/builtin-monitors.ts +3 -0
- package/src/services/alerting/builtin-source.ts +23 -0
- package/src/services/audit/index.test.ts +2 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/public-dns-source.ts +55 -0
- package/src/services/audit/public-dns.test.ts +209 -0
- package/src/services/audit/public-dns.ts +286 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/backup-metadata.ts +17 -0
- package/src/services/backup-staging.test.ts +98 -0
- package/src/services/backup-staging.ts +73 -1
- package/src/services/backup-sweep.test.ts +15 -0
- package/src/services/backup-sweep.ts +17 -1
- package/src/services/bus-interview-park.test.ts +179 -0
- package/src/services/bus-interview.ts +13 -8
- package/src/services/dns-registrations.test.ts +78 -16
- package/src/services/dns-registrations.ts +107 -19
- package/src/services/fleet-checks.test.ts +47 -1
- package/src/services/fleet-checks.ts +36 -4
- package/src/services/interview-errors.ts +24 -7
- package/src/services/module-subscriptions.test.ts +9 -0
- package/src/services/public-dns-probe.test.ts +81 -0
- package/src/services/public-dns-probe.ts +156 -0
- package/src/services/remote-responder.test.ts +33 -20
- package/src/services/remote-responder.ts +10 -6
- package/src/services/update/orchestrator.test.ts +2 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parked-session registry for `api-serve` (celilo#609).
|
|
3
|
+
*
|
|
4
|
+
* A command that parks on an unanswerable question must outlive the ssh session
|
|
5
|
+
* that started it, so a *different* responder can answer later and the caller
|
|
6
|
+
* can come back for the outcome. Two consequences shape this file:
|
|
7
|
+
*
|
|
8
|
+
* 1. **The registry is on disk, not in memory.** Each ssh connection is its own
|
|
9
|
+
* `api-serve` process, so the process that later handles `attach` is not the
|
|
10
|
+
* one holding the parked child. A directory under the data dir is the
|
|
11
|
+
* handoff: metadata in `session.json`, the command's output appended to
|
|
12
|
+
* `output.ndjson`. The attaching process replays the file and tails it.
|
|
13
|
+
* 2. **The record is retained after the session ends.** A repeatedly-parking
|
|
14
|
+
* command is only detectable if the abandonments are still there to see —
|
|
15
|
+
* the same reason `module operations` keeps its rows.
|
|
16
|
+
*
|
|
17
|
+
* ponytail: files + polling rather than a socket/IPC server. Same process model,
|
|
18
|
+
* a fraction of the machinery, and the buffered output has to be durable across
|
|
19
|
+
* processes anyway. Move to a socket only if attach latency becomes a problem.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import {
|
|
23
|
+
appendFileSync,
|
|
24
|
+
existsSync,
|
|
25
|
+
mkdirSync,
|
|
26
|
+
readFileSync,
|
|
27
|
+
readdirSync,
|
|
28
|
+
writeFileSync,
|
|
29
|
+
} from 'node:fs';
|
|
30
|
+
import { join } from 'node:path';
|
|
31
|
+
import type { ServerMessage } from '@celilo/core';
|
|
32
|
+
import { defineEvents, openBus } from '@celilo/event-bus';
|
|
33
|
+
import { z } from 'zod';
|
|
34
|
+
import { getDataDir } from '../config/paths';
|
|
35
|
+
|
|
36
|
+
const NO_SCHEMAS = defineEvents({});
|
|
37
|
+
|
|
38
|
+
/** How long a parked session may hold its child before the reaper abandons it. */
|
|
39
|
+
export const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
|
|
40
|
+
|
|
41
|
+
/** Lines of buffered output an attaching client is replayed at most. */
|
|
42
|
+
const REPLAY_LIMIT = 2000;
|
|
43
|
+
|
|
44
|
+
export const SessionStateSchema = z.enum(['running', 'parked', 'finished', 'abandoned']);
|
|
45
|
+
export type SessionState = z.infer<typeof SessionStateSchema>;
|
|
46
|
+
|
|
47
|
+
export const SessionRecordSchema = z.object({
|
|
48
|
+
sessionId: z.string(),
|
|
49
|
+
principal: z.string(),
|
|
50
|
+
argv: z.array(z.string()),
|
|
51
|
+
startedAt: z.number(),
|
|
52
|
+
expiresAt: z.number(),
|
|
53
|
+
state: SessionStateSchema,
|
|
54
|
+
/** Bus event id of the question this session is parked on, when parked. */
|
|
55
|
+
parkedEventId: z.string().nullable(),
|
|
56
|
+
/** Bus event *type* of that question — what the reaper replies to. */
|
|
57
|
+
parkedEventType: z.string().nullable(),
|
|
58
|
+
question: z.string().nullable(),
|
|
59
|
+
/** The parked question's `<scope>.<key>`. */
|
|
60
|
+
questionKey: z.string().nullable(),
|
|
61
|
+
});
|
|
62
|
+
export type SessionRecord = z.infer<typeof SessionRecordSchema>;
|
|
63
|
+
|
|
64
|
+
export function getSessionsDir(): string {
|
|
65
|
+
return join(getDataDir(), 'api-sessions');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function sessionDir(sessionId: string): string {
|
|
69
|
+
return join(getSessionsDir(), sessionId);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function recordPath(sessionId: string): string {
|
|
73
|
+
return join(sessionDir(sessionId), 'session.json');
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function outputPath(sessionId: string): string {
|
|
77
|
+
return join(sessionDir(sessionId), 'output.ndjson');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function readSession(sessionId: string): SessionRecord | null {
|
|
81
|
+
const path = recordPath(sessionId);
|
|
82
|
+
if (!existsSync(path)) return null;
|
|
83
|
+
// Untrusted only in the sense of "written by another process" — validate it
|
|
84
|
+
// rather than trusting the shape (Rule 3.7).
|
|
85
|
+
const parsed = SessionRecordSchema.safeParse(JSON.parse(readFileSync(path, 'utf-8')));
|
|
86
|
+
return parsed.success ? parsed.data : null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function listSessions(): SessionRecord[] {
|
|
90
|
+
const dir = getSessionsDir();
|
|
91
|
+
if (!existsSync(dir)) return [];
|
|
92
|
+
return readdirSync(dir)
|
|
93
|
+
.map(readSession)
|
|
94
|
+
.filter((s): s is SessionRecord => s !== null)
|
|
95
|
+
.sort((a, b) => b.startedAt - a.startedAt);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The session parked on a given bus query, if one is. */
|
|
99
|
+
export function sessionParkedOn(eventId: string): SessionRecord | null {
|
|
100
|
+
return listSessions().find((s) => s.state === 'parked' && s.parkedEventId === eventId) ?? null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function write(record: SessionRecord): void {
|
|
104
|
+
mkdirSync(sessionDir(record.sessionId), { recursive: true });
|
|
105
|
+
writeFileSync(recordPath(record.sessionId), `${JSON.stringify(record, null, 2)}\n`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* A live session's handle: the writer side of the registry. Held by the
|
|
110
|
+
* `api-serve` process running the command.
|
|
111
|
+
*/
|
|
112
|
+
export class SessionWriter {
|
|
113
|
+
private record: SessionRecord;
|
|
114
|
+
|
|
115
|
+
private constructor(record: SessionRecord) {
|
|
116
|
+
this.record = record;
|
|
117
|
+
write(record);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
static create(opts: {
|
|
121
|
+
principal: string;
|
|
122
|
+
argv: string[];
|
|
123
|
+
ttlMs?: number;
|
|
124
|
+
now?: number;
|
|
125
|
+
}): SessionWriter {
|
|
126
|
+
const now = opts.now ?? Date.now();
|
|
127
|
+
return new SessionWriter({
|
|
128
|
+
sessionId: crypto.randomUUID(),
|
|
129
|
+
principal: opts.principal,
|
|
130
|
+
argv: opts.argv,
|
|
131
|
+
startedAt: now,
|
|
132
|
+
expiresAt: now + (opts.ttlMs ?? DEFAULT_SESSION_TTL_MS),
|
|
133
|
+
state: 'running',
|
|
134
|
+
parkedEventId: null,
|
|
135
|
+
parkedEventType: null,
|
|
136
|
+
question: null,
|
|
137
|
+
questionKey: null,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
get id(): string {
|
|
142
|
+
return this.record.sessionId;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
get current(): SessionRecord {
|
|
146
|
+
return this.record;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Append a message to the buffer so a client attaching later sees it. */
|
|
150
|
+
append(msg: ServerMessage): void {
|
|
151
|
+
appendFileSync(outputPath(this.record.sessionId), `${JSON.stringify(msg)}\n`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
park(opts: { eventId: string; eventType: string; question: string; questionKey?: string }): void {
|
|
155
|
+
this.record = {
|
|
156
|
+
...this.record,
|
|
157
|
+
state: 'parked',
|
|
158
|
+
parkedEventId: opts.eventId,
|
|
159
|
+
parkedEventType: opts.eventType,
|
|
160
|
+
question: opts.question,
|
|
161
|
+
questionKey: opts.questionKey ?? null,
|
|
162
|
+
};
|
|
163
|
+
write(this.record);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** The question was answered (by anyone) — the command is running again. */
|
|
167
|
+
unpark(): void {
|
|
168
|
+
if (this.record.state !== 'parked') return;
|
|
169
|
+
this.record = {
|
|
170
|
+
...this.record,
|
|
171
|
+
state: 'running',
|
|
172
|
+
parkedEventId: null,
|
|
173
|
+
parkedEventType: null,
|
|
174
|
+
question: null,
|
|
175
|
+
questionKey: null,
|
|
176
|
+
};
|
|
177
|
+
write(this.record);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* `abandoned` outranks `finished`: the command exits *because* it was
|
|
182
|
+
* abandoned, and that exit must not overwrite why. Retained either way.
|
|
183
|
+
*/
|
|
184
|
+
finish(state: 'finished' | 'abandoned'): void {
|
|
185
|
+
if (this.record.state === 'abandoned') return;
|
|
186
|
+
this.record = { ...this.record, state };
|
|
187
|
+
write(this.record);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Answer a parked session's outstanding query as `abandoned` and retain the
|
|
193
|
+
* record as history.
|
|
194
|
+
*
|
|
195
|
+
* The reply is what releases everything the parked command holds: the child
|
|
196
|
+
* resumes, throws `InterviewAbandonedError`, and unwinds — releasing its own
|
|
197
|
+
* module-operation lock on the way out, exactly as any other failure does. That
|
|
198
|
+
* is the whole point of answering rather than killing.
|
|
199
|
+
*
|
|
200
|
+
* ponytail: no forced kill if the child ignores its own unwinding. Add one only
|
|
201
|
+
* once a real command is seen to survive an abandoned answer.
|
|
202
|
+
*/
|
|
203
|
+
export function abandonSession(
|
|
204
|
+
record: SessionRecord,
|
|
205
|
+
opts: { busDbPath: string; reason: string; emittedBy?: string },
|
|
206
|
+
): boolean {
|
|
207
|
+
if (record.parkedEventId && record.parkedEventType) {
|
|
208
|
+
const bus = openBus({ dbPath: opts.busDbPath, events: NO_SCHEMAS });
|
|
209
|
+
try {
|
|
210
|
+
// Someone decided it while we were on our way to reap. Abandoning now
|
|
211
|
+
// would overwrite a real answer with "nobody decided" — the exact
|
|
212
|
+
// fabrication this whole change exists to remove, only in reverse.
|
|
213
|
+
if (bus.repliesFor(Number(record.parkedEventId)).length > 0) return false;
|
|
214
|
+
bus.emitRaw(
|
|
215
|
+
`${record.parkedEventType}.reply`,
|
|
216
|
+
{ abandoned: { reason: opts.reason } },
|
|
217
|
+
{
|
|
218
|
+
replyFor: Number(record.parkedEventId),
|
|
219
|
+
emittedBy: opts.emittedBy ?? 'api-session-reaper',
|
|
220
|
+
},
|
|
221
|
+
);
|
|
222
|
+
} finally {
|
|
223
|
+
bus.close();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
write({ ...record, state: 'abandoned' });
|
|
227
|
+
return true;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Is this session still waiting on a decision *according to the bus*?
|
|
232
|
+
*
|
|
233
|
+
* The record's `state` is the owning process's view and can lag: the answer
|
|
234
|
+
* arrives on the bus from a third party, which the owner never sees. The bus is
|
|
235
|
+
* the source of truth for whether a question stands — ask it, don't infer.
|
|
236
|
+
*/
|
|
237
|
+
export function stillParked(record: SessionRecord, busDbPath: string): boolean {
|
|
238
|
+
if (record.state !== 'parked' || !record.parkedEventId) return false;
|
|
239
|
+
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
240
|
+
try {
|
|
241
|
+
return bus.repliesFor(Number(record.parkedEventId)).length === 0;
|
|
242
|
+
} finally {
|
|
243
|
+
bus.close();
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Why a reaped session's question was never decided. */
|
|
248
|
+
export function expiryReason(record: SessionRecord): string {
|
|
249
|
+
const what = record.question ?? 'the question';
|
|
250
|
+
return `Session ${record.sessionId} expired with "${what}" still unanswered — nobody decided it.`;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Abandon every parked session past its TTL. Run by the process that owns a
|
|
255
|
+
* parked child on its own timer, and again at `api-serve` startup so a session
|
|
256
|
+
* whose owner died (reboot, OOM) doesn't sit parked forever — the failure mode
|
|
257
|
+
* that left a `module deploy` holding every backup lock on the fleet for 20 days.
|
|
258
|
+
*/
|
|
259
|
+
export function reapExpiredSessions(opts: { busDbPath: string; now?: number }): SessionRecord[] {
|
|
260
|
+
const now = opts.now ?? Date.now();
|
|
261
|
+
const expired = listSessions().filter((s) => s.state === 'parked' && s.expiresAt <= now);
|
|
262
|
+
for (const record of expired) {
|
|
263
|
+
abandonSession(record, { busDbPath: opts.busDbPath, reason: expiryReason(record) });
|
|
264
|
+
}
|
|
265
|
+
return expired;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Buffered messages an attaching client should be replayed. */
|
|
269
|
+
export function replayOutput(
|
|
270
|
+
sessionId: string,
|
|
271
|
+
fromLine = 0,
|
|
272
|
+
): { messages: string[]; next: number } {
|
|
273
|
+
const path = outputPath(sessionId);
|
|
274
|
+
if (!existsSync(path)) return { messages: [], next: fromLine };
|
|
275
|
+
const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
|
|
276
|
+
const start = Math.max(fromLine, lines.length - REPLAY_LIMIT);
|
|
277
|
+
return { messages: lines.slice(start), next: lines.length };
|
|
278
|
+
}
|
|
@@ -17,13 +17,12 @@ import {
|
|
|
17
17
|
findBackupEligibleModules,
|
|
18
18
|
isBackupDue,
|
|
19
19
|
} from '../../services/backup-create';
|
|
20
|
-
import { failBackup, getBackup } from '../../services/backup-metadata';
|
|
20
|
+
import { failBackup, getBackup, listInProgressBackups } from '../../services/backup-metadata';
|
|
21
21
|
import { pruneBackupsForModule } from '../../services/backup-retention';
|
|
22
22
|
import {
|
|
23
|
-
ABANDONED_BACKUP_MESSAGE,
|
|
24
23
|
STAGING_PREFIX,
|
|
25
|
-
impliesAbandonedRecord,
|
|
26
24
|
reapOrphanedStaging,
|
|
25
|
+
resolveAbandonedBackups,
|
|
27
26
|
} from '../../services/backup-staging';
|
|
28
27
|
import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
|
|
29
28
|
import { isPidRunnable } from '../../services/module-operations';
|
|
@@ -59,18 +58,30 @@ function reapStaging() {
|
|
|
59
58
|
now: () => Date.now(),
|
|
60
59
|
});
|
|
61
60
|
|
|
62
|
-
for (const reclaimed of report.reclaimed) {
|
|
63
|
-
if (impliesAbandonedRecord(reclaimed.reason)) {
|
|
64
|
-
failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
61
|
return report;
|
|
69
62
|
}
|
|
70
63
|
|
|
64
|
+
/**
|
|
65
|
+
* Correct every record that still claims to be running, staging or no staging.
|
|
66
|
+
*
|
|
67
|
+
* This replaces deriving the fix-up from what the reaper reclaimed. That
|
|
68
|
+
* version only ever saw records whose directory still existed, so anything
|
|
69
|
+
* cleared by a reboot or by hand stayed `in_progress` forever (#616).
|
|
70
|
+
*/
|
|
71
|
+
function resolveAbandonedRecords() {
|
|
72
|
+
return resolveAbandonedBackups({
|
|
73
|
+
listInProgress: () =>
|
|
74
|
+
listInProgressBackups().map((b) => ({ id: b.id, pid: b.pid, startedAt: b.startedAt })),
|
|
75
|
+
isPidRunnable,
|
|
76
|
+
fail: (id, message) => failBackup(id, message),
|
|
77
|
+
now: () => Date.now(),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
71
81
|
export async function handleBackupSweep(): Promise<CommandResult> {
|
|
72
82
|
const report = await runBackupSweep({
|
|
73
83
|
reapStaging,
|
|
84
|
+
resolveAbandonedRecords,
|
|
74
85
|
listEligible: () =>
|
|
75
86
|
findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
|
|
76
87
|
isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
|
|
@@ -117,6 +128,11 @@ function formatReport(report: BackupSweepReport): string {
|
|
|
117
128
|
lines.push(` ${reclaimed.recordId} (${reclaimed.reason})`);
|
|
118
129
|
}
|
|
119
130
|
}
|
|
131
|
+
if (report.records.resolved.length > 0) {
|
|
132
|
+
lines.push(
|
|
133
|
+
` corrected ${report.records.resolved.length} record(s) that still claimed to be running`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
120
136
|
for (const moduleId of report.skippedLocked) {
|
|
121
137
|
lines.push(` skipped ${moduleId}: another operation is in flight — retrying next tick`);
|
|
122
138
|
}
|
package/src/cli/commands/dns.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* `registrations` lists the dns_registrations ledger: every (provider,
|
|
5
5
|
* fqdn) the framework has registered via dns_registrar.registerHost,
|
|
6
|
-
* with
|
|
6
|
+
* with every module that depends on it and when it was last re-asserted
|
|
7
7
|
* by the provider's refresh_registrations hook. Read-only; names only
|
|
8
8
|
* (module ids), never UUIDs. See
|
|
9
9
|
* designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B2b).
|
|
@@ -40,12 +40,16 @@ export async function handleDnsRegistrations(
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
const now = new Date();
|
|
43
|
-
|
|
43
|
+
// No address column: the ledger stores none. What a name actually resolves
|
|
44
|
+
// to publicly is what `celilo system audit`'s public_dns check reports, from
|
|
45
|
+
// off-fleet — reading it back out of celilo's own table is what made a
|
|
46
|
+
// nine-day outage invisible (design.md D1).
|
|
47
|
+
const header = ['FQDN', 'KIND', 'PROVIDER', 'CONSUMERS', 'REGISTERED', 'REFRESHED'];
|
|
44
48
|
const table = rows.map((r) => [
|
|
45
49
|
r.fqdn,
|
|
46
|
-
r.
|
|
50
|
+
r.companion ? 'companion' : 'declared',
|
|
47
51
|
r.providerModuleId,
|
|
48
|
-
r.
|
|
52
|
+
r.consumerModuleIds.join(', ') || '—',
|
|
49
53
|
formatAge(r.registeredAt, now),
|
|
50
54
|
formatAge(r.refreshedAt, now),
|
|
51
55
|
]);
|
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* status bus.health() as JSON
|
|
10
10
|
* tail recent events
|
|
11
11
|
* list-subscribers persistent subscribers
|
|
12
|
-
* list-pending pending deliveries
|
|
12
|
+
* list-pending pending deliveries (subscriber fan-out — NOT questions)
|
|
13
|
+
* list-unanswered interview questions nobody has answered yet
|
|
13
14
|
* drain process pending deliveries once
|
|
14
15
|
* run long-running dispatcher (foreground; SIGINT to stop)
|
|
15
16
|
* emit <type> [json] emit an event (operator/test path; bypasses schema)
|
|
@@ -30,6 +31,7 @@ import {
|
|
|
30
31
|
runDispatcher,
|
|
31
32
|
} from '@celilo/event-bus';
|
|
32
33
|
import { eq } from 'drizzle-orm';
|
|
34
|
+
import { sessionParkedOn } from '../../api/sessions';
|
|
33
35
|
import { getEventBusPath, shortenPath } from '../../config/paths';
|
|
34
36
|
import { getDb } from '../../db/client';
|
|
35
37
|
import { modules } from '../../db/schema';
|
|
@@ -150,7 +152,10 @@ export async function handleEventsRunHook(args: string[]): Promise<CommandResult
|
|
|
150
152
|
};
|
|
151
153
|
|
|
152
154
|
const logger = createConsoleLogger(moduleId, sub.hook);
|
|
153
|
-
const result = await runNamedHook(moduleId, sub.hook as HookName, db, logger, {
|
|
155
|
+
const result = await runNamedHook(moduleId, sub.hook as HookName, db, logger, {
|
|
156
|
+
inputs,
|
|
157
|
+
timeoutMs: sub.timeout_ms,
|
|
158
|
+
});
|
|
154
159
|
|
|
155
160
|
if (result.notDefined) {
|
|
156
161
|
return { success: false, error: `Module '${moduleId}' declares no '${sub.hook}' hook to run` };
|
|
@@ -241,6 +246,57 @@ export async function handleEventsListPending(
|
|
|
241
246
|
}
|
|
242
247
|
}
|
|
243
248
|
|
|
249
|
+
/** One unanswered interview question, as `events list-unanswered` reports it. */
|
|
250
|
+
export interface UnansweredInterview {
|
|
251
|
+
eventId: number;
|
|
252
|
+
type: string;
|
|
253
|
+
family: InterviewFamily;
|
|
254
|
+
/** `<scope>.<key>` — the identity a responder pre-stages an answer under. */
|
|
255
|
+
key: string;
|
|
256
|
+
question: string;
|
|
257
|
+
ageMs: number;
|
|
258
|
+
/** The parked api-serve session waiting on this answer, if any. */
|
|
259
|
+
sessionId: string | null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* `celilo events list-unanswered` — interview queries with no correlated reply:
|
|
264
|
+
* what is waiting on a decision right now.
|
|
265
|
+
*
|
|
266
|
+
* The instrument celilo#609 lacked. `events list-pending` was reached for and
|
|
267
|
+
* silently answered a different question (it reads subscriber *deliveries*), so
|
|
268
|
+
* a parked command looked like no command at all. Non-empty here for as long as
|
|
269
|
+
* something is parked is the recurrence gate for that whole class of bug.
|
|
270
|
+
*/
|
|
271
|
+
export async function handleEventsListUnanswered(
|
|
272
|
+
_args: string[],
|
|
273
|
+
flags: Record<string, string | boolean>,
|
|
274
|
+
): Promise<CommandResult> {
|
|
275
|
+
const bus = openCliBus();
|
|
276
|
+
try {
|
|
277
|
+
const limit = flags.limit ? Number(flags.limit) : 50;
|
|
278
|
+
const now = Date.now();
|
|
279
|
+
const rows: UnansweredInterview[] = [];
|
|
280
|
+
for (const event of bus.unansweredQueries({ limit })) {
|
|
281
|
+
const family = interviewFamily(event.type);
|
|
282
|
+
if (!family) continue; // e.g. responder.probe — not a question for an operator.
|
|
283
|
+
const payload = (event.payload ?? {}) as { message?: string; description?: string };
|
|
284
|
+
rows.push({
|
|
285
|
+
eventId: event.id,
|
|
286
|
+
type: event.type,
|
|
287
|
+
family,
|
|
288
|
+
key: event.type.slice(`${family}.required.`.length),
|
|
289
|
+
question: payload.message ?? payload.description ?? event.type,
|
|
290
|
+
ageMs: now - event.emittedAt,
|
|
291
|
+
sessionId: sessionParkedOn(String(event.id))?.sessionId ?? null,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
return jsonResult(rows);
|
|
295
|
+
} finally {
|
|
296
|
+
bus.close();
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
244
300
|
export async function handleEventsDrain(
|
|
245
301
|
_args: string[],
|
|
246
302
|
flags: Record<string, string | boolean>,
|
|
@@ -412,9 +468,12 @@ function interviewFamily(type: string): InterviewFamily | null {
|
|
|
412
468
|
/**
|
|
413
469
|
* `celilo events reply <query-event-id> <value-json>` — answer ONE pending
|
|
414
470
|
* interview query by its event id. The one-shot reply primitive a
|
|
415
|
-
* `claude-config-responder` uses:
|
|
416
|
-
* `events
|
|
417
|
-
*
|
|
471
|
+
* `claude-config-responder` uses: find the question with
|
|
472
|
+
* `celilo events list-unanswered`, ask the operator, emit the answer here.
|
|
473
|
+
* (It used to say "read the log with `events tail --type '…'`" — hand-scraping
|
|
474
|
+
* the log because no command listed unanswered questions. `list-unanswered` is
|
|
475
|
+
* that command; it also names the parked session, which `tail` cannot.)
|
|
476
|
+
* Unlike `events respond` (which must be subscribed BEFORE the query is
|
|
418
477
|
* emitted — bus watches don't replay history) this looks the query up by id
|
|
419
478
|
* and emits a correlated reply carrying `replyFor`, which plain `events emit`
|
|
420
479
|
* can't set.
|
|
@@ -23,7 +23,7 @@ import type { ModuleManifest } from '../../manifest/schema';
|
|
|
23
23
|
import { cleanupTempDir, extractPackage } from '../../module/packaging/extract';
|
|
24
24
|
import { RegistryClient } from '../../registry/client';
|
|
25
25
|
import { askConfirm, withInterviewSession } from '../../services/bus-interview';
|
|
26
|
-
import { InterviewUnansweredError } from '../../services/interview-errors';
|
|
26
|
+
import { InterviewAbandonedError, InterviewUnansweredError } from '../../services/interview-errors';
|
|
27
27
|
import { getFlag } from '../parser';
|
|
28
28
|
import { log } from '../prompts';
|
|
29
29
|
import type { CommandResult } from '../types';
|
|
@@ -499,6 +499,7 @@ async function runRegistrySweep(
|
|
|
499
499
|
let appliedBreaking = 0;
|
|
500
500
|
let declinedBreaking = 0;
|
|
501
501
|
const unanswered: Array<{ moduleId: string; error: string }> = [];
|
|
502
|
+
const abandoned: Array<{ moduleId: string; error: string }> = [];
|
|
502
503
|
|
|
503
504
|
if (breaking.length > 0) {
|
|
504
505
|
log.info('\nBreaking updates available — review required (semver-major bump):');
|
|
@@ -521,15 +522,25 @@ async function runRegistrySweep(
|
|
|
521
522
|
}),
|
|
522
523
|
);
|
|
523
524
|
} catch (err) {
|
|
524
|
-
//
|
|
525
|
-
//
|
|
526
|
-
//
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
525
|
+
// Neither of these is a decline, and they are not each other either:
|
|
526
|
+
// UNANSWERED = no responder was listening at all (fail-fast probe);
|
|
527
|
+
// ABANDONED = the question stood and its parked session expired.
|
|
528
|
+
// Record and keep going so updates already applied aren't thrown away.
|
|
529
|
+
if (err instanceof InterviewUnansweredError) {
|
|
530
|
+
unanswered.push({ moduleId: plan.moduleId, error: err.message });
|
|
531
|
+
console.log(
|
|
532
|
+
` ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion} (major, UNANSWERED)`,
|
|
533
|
+
);
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
if (err instanceof InterviewAbandonedError) {
|
|
537
|
+
abandoned.push({ moduleId: plan.moduleId, error: err.message });
|
|
538
|
+
console.log(
|
|
539
|
+
` ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion} (major, ABANDONED)`,
|
|
540
|
+
);
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
throw err;
|
|
533
544
|
}
|
|
534
545
|
if (!proceed) {
|
|
535
546
|
declinedBreaking++;
|
|
@@ -572,6 +583,15 @@ async function runRegistrySweep(
|
|
|
572
583
|
)}). Re-run with a responder attached, or pre-stage the answer under "module-upgrade:<module>.apply_breaking".`,
|
|
573
584
|
);
|
|
574
585
|
}
|
|
586
|
+
if (abandoned.length > 0) {
|
|
587
|
+
summary.push(
|
|
588
|
+
`Skipped ${abandoned.length} breaking update(s) — NOT declined: the confirmation was never decided and its session expired (${abandoned
|
|
589
|
+
.map((a) => a.moduleId)
|
|
590
|
+
.join(
|
|
591
|
+
', ',
|
|
592
|
+
)}). Answer it next time with "celilo events list-unanswered" + "celilo events reply <id> <value>".`,
|
|
593
|
+
);
|
|
594
|
+
}
|
|
575
595
|
if (notInRegistry.length > 0) {
|
|
576
596
|
summary.push(`Not in registry (${notInRegistry.length}): ${notInRegistry.join(', ')}`);
|
|
577
597
|
}
|
|
@@ -580,7 +600,7 @@ async function runRegistrySweep(
|
|
|
580
600
|
`Registry errors (${errored.length}): ${errored.map((e) => `${e.moduleId} — ${e.error}`).join('; ')}`,
|
|
581
601
|
);
|
|
582
602
|
}
|
|
583
|
-
if (failed.length > 0 || unanswered.length > 0) {
|
|
603
|
+
if (failed.length > 0 || unanswered.length > 0 || abandoned.length > 0) {
|
|
584
604
|
const detail: string[] = [];
|
|
585
605
|
if (failed.length > 0) {
|
|
586
606
|
detail.push('', 'Failures:', ...failed.map((f) => ` ${f.moduleId}: ${f.error}`));
|
|
@@ -588,6 +608,9 @@ async function runRegistrySweep(
|
|
|
588
608
|
if (unanswered.length > 0) {
|
|
589
609
|
detail.push('', 'Unanswered:', ...unanswered.map((u) => ` ${u.moduleId}: ${u.error}`));
|
|
590
610
|
}
|
|
611
|
+
if (abandoned.length > 0) {
|
|
612
|
+
detail.push('', 'Abandoned:', ...abandoned.map((a) => ` ${a.moduleId}: ${a.error}`));
|
|
613
|
+
}
|
|
591
614
|
return { success: false, error: [...summary, ...detail].join('\n') };
|
|
592
615
|
}
|
|
593
616
|
return { success: true, message: summary.join('\n') };
|
|
@@ -46,6 +46,10 @@ import {
|
|
|
46
46
|
} from '../../services/audit/cli-version';
|
|
47
47
|
import type { MachineReachableResult } from '../../services/audit/machines-reachable';
|
|
48
48
|
import type { ModuleVersionFetcher } from '../../services/audit/module-versions';
|
|
49
|
+
import {
|
|
50
|
+
loadPublicDnsEvidence,
|
|
51
|
+
loadPublicDnsRecords,
|
|
52
|
+
} from '../../services/audit/public-dns-source';
|
|
49
53
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
50
54
|
import type { SecretCheckResult } from '../../services/audit/secrets-decryptable';
|
|
51
55
|
import type { ServiceCredentialsResult } from '../../services/audit/services-credentials';
|
|
@@ -56,6 +60,7 @@ import { collectFirewallReach } from '../../services/firewall-reach';
|
|
|
56
60
|
import { runAllHealthChecks } from '../../services/health-runner';
|
|
57
61
|
import { probeMachines } from '../../services/machine-probe';
|
|
58
62
|
import { parseStoredConfigValue } from '../../services/module-config';
|
|
63
|
+
import { createPublicDnsProbe, loadPublicDnsProbeSettings } from '../../services/public-dns-probe';
|
|
59
64
|
import { buildTerraformEnvForModule } from '../../services/terraform-env';
|
|
60
65
|
import { hasFlag } from '../parser';
|
|
61
66
|
import type { CommandResult } from '../types';
|
|
@@ -381,6 +386,16 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
381
386
|
secretsDecryptable: { results: secretResults },
|
|
382
387
|
servicesReachable: { results: serviceReachableResults },
|
|
383
388
|
machinesReachable: { results: machineReachableResults },
|
|
389
|
+
// The only check here whose vantage point is OUTSIDE the fleet. Its
|
|
390
|
+
// undetermined counters are read but not written from this path: an
|
|
391
|
+
// operator-run audit is not a run of a schedule, so counting it towards
|
|
392
|
+
// "N consecutive runs found no evidence" would misreport how long the
|
|
393
|
+
// fleet has been unverifiable. The monitor sweep owns that (D2).
|
|
394
|
+
publicDns: {
|
|
395
|
+
records: loadPublicDnsRecords(db),
|
|
396
|
+
probe: createPublicDnsProbe(loadPublicDnsProbeSettings(db)),
|
|
397
|
+
evidence: loadPublicDnsEvidence(db),
|
|
398
|
+
},
|
|
384
399
|
// Reads the record the poller already writes — this check never performs a
|
|
385
400
|
// read of its own. One that did would drain the queue and eat the
|
|
386
401
|
// acknowledgement it exists to protect (#541).
|
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
2
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
3
|
-
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { closeDb } from '../../db/client';
|
|
6
|
+
import { closeDb, findMigrationsFolder } from '../../db/client';
|
|
7
7
|
import { handleSystemMigrate } from './system-migrate';
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* The newest migration, read from the journal rather than written down here.
|
|
11
|
+
*
|
|
12
|
+
* These assertions used to name `0019_backup_pid` literally, so every
|
|
13
|
+
* subsequent migration broke a test that has nothing to do with it. What is
|
|
14
|
+
* under test is that `--status` REPORTS the head and any gap below it, not
|
|
15
|
+
* which migration happens to be head today.
|
|
16
|
+
*/
|
|
17
|
+
function latestMigrationTag(): string {
|
|
18
|
+
const journal = JSON.parse(
|
|
19
|
+
readFileSync(join(findMigrationsFolder(), 'meta', '_journal.json'), 'utf8'),
|
|
20
|
+
) as { entries: { tag: string }[] };
|
|
21
|
+
const tag = journal.entries.at(-1)?.tag;
|
|
22
|
+
if (!tag) throw new Error('Migration journal is empty');
|
|
23
|
+
return tag;
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
describe('handleSystemMigrate', () => {
|
|
10
27
|
let dir: string;
|
|
11
28
|
|
|
@@ -51,7 +68,7 @@ describe('handleSystemMigrate', () => {
|
|
|
51
68
|
expect(result.success).toBe(true);
|
|
52
69
|
if (result.success) {
|
|
53
70
|
expect(result.message).toMatch(/Applied migrations: \d+/);
|
|
54
|
-
expect(result.message).toContain(
|
|
71
|
+
expect(result.message).toContain(latestMigrationTag());
|
|
55
72
|
expect(result.message).toContain('Pending: none');
|
|
56
73
|
expect(result.message).toContain('columns');
|
|
57
74
|
}
|
|
@@ -61,6 +78,7 @@ describe('handleSystemMigrate', () => {
|
|
|
61
78
|
await handleSystemMigrate();
|
|
62
79
|
closeDb();
|
|
63
80
|
// Rewind one migration, the way an upgrade that never ran would look.
|
|
81
|
+
const head = latestMigrationTag();
|
|
64
82
|
const raw = new Database(process.env.CELILO_DB_PATH as string);
|
|
65
83
|
raw.run(
|
|
66
84
|
'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
|
|
@@ -75,7 +93,10 @@ describe('handleSystemMigrate', () => {
|
|
|
75
93
|
|
|
76
94
|
expect(result.success).toBe(false);
|
|
77
95
|
if (!result.success) {
|
|
78
|
-
|
|
96
|
+
// The rewound migration is named as pending…
|
|
97
|
+
expect(result.error).toContain(head);
|
|
98
|
+
// …and the dropped COLUMN is reported independently, which is the
|
|
99
|
+
// thing a table count cannot see (celilo#604).
|
|
79
100
|
expect(result.error).toContain('backups.pid');
|
|
80
101
|
}
|
|
81
102
|
|
|
@@ -28,6 +28,7 @@ import { RegistryClient } from '../../registry/client';
|
|
|
28
28
|
import { runAudit } from '../../services/audit';
|
|
29
29
|
import { loadAbandonedOperations } from '../../services/audit/abandoned-operations';
|
|
30
30
|
import { fetchLatestCliVersion } from '../../services/audit/cli-version';
|
|
31
|
+
import { unusedPublicDnsProbe } from '../../services/audit/public-dns';
|
|
31
32
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
32
33
|
import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
|
|
33
34
|
import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner';
|
|
@@ -587,6 +588,10 @@ export async function handleSystemUpdate(
|
|
|
587
588
|
secretsDecryptable: { results: [] },
|
|
588
589
|
servicesReachable: { results: [] },
|
|
589
590
|
machinesReachable: { results: [] },
|
|
591
|
+
// Public reachability needs a network round trip per name; the update
|
|
592
|
+
// flow's partial audit does no probing. `system audit` and the scheduled
|
|
593
|
+
// monitor own this check.
|
|
594
|
+
publicDns: { records: [], probe: unusedPublicDnsProbe },
|
|
590
595
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
591
596
|
trustedSources: { firewalls: [] },
|
|
592
597
|
};
|