@celilo/cli 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/remote-client.test.ts +34 -12
- package/src/api/serve.ts +234 -38
- package/src/api/sessions.test.ts +196 -0
- package/src/api/sessions.ts +278 -0
- package/src/cli/commands/backup-sweep.ts +25 -9
- package/src/cli/commands/events.ts +60 -4
- package/src/cli/commands/module-update.ts +34 -11
- package/src/cli/completion.ts +1 -0
- package/src/cli/index.ts +22 -2
- package/src/services/backup-metadata.ts +17 -0
- package/src/services/backup-staging.test.ts +98 -0
- package/src/services/backup-staging.ts +73 -1
- package/src/services/backup-sweep.test.ts +15 -0
- package/src/services/backup-sweep.ts +17 -1
- package/src/services/bus-interview-park.test.ts +179 -0
- package/src/services/bus-interview.ts +13 -8
- package/src/services/interview-errors.ts +24 -7
- package/src/services/remote-responder.test.ts +33 -20
- package/src/services/remote-responder.ts +10 -6
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import {
|
|
3
|
+
ABANDONED_BACKUP_MESSAGE,
|
|
4
|
+
type InProgressBackup,
|
|
3
5
|
type ReapStagingDeps,
|
|
6
|
+
type ResolveAbandonedDeps,
|
|
4
7
|
STAGING_PREFIX,
|
|
5
8
|
STAGING_TTL_MS,
|
|
6
9
|
type StagingOwner,
|
|
7
10
|
reapOrphanedStaging,
|
|
11
|
+
resolveAbandonedBackups,
|
|
8
12
|
stagingDirFor,
|
|
9
13
|
} from './backup-staging';
|
|
10
14
|
|
|
@@ -132,3 +136,97 @@ describe('reapOrphanedStaging', () => {
|
|
|
132
136
|
expect(report.kept).toEqual([locked]);
|
|
133
137
|
});
|
|
134
138
|
});
|
|
139
|
+
|
|
140
|
+
function recordDeps(
|
|
141
|
+
records: InProgressBackup[],
|
|
142
|
+
runnablePids: number[] = [],
|
|
143
|
+
): ResolveAbandonedDeps & { failed: Array<{ id: string; message: string }> } {
|
|
144
|
+
const failed: Array<{ id: string; message: string }> = [];
|
|
145
|
+
return {
|
|
146
|
+
failed,
|
|
147
|
+
listInProgress: () => records,
|
|
148
|
+
isPidRunnable: (pid) => runnablePids.includes(pid),
|
|
149
|
+
fail: (id, message) => failed.push({ id, message }),
|
|
150
|
+
now: () => NOW,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function record(over: Partial<InProgressBackup> = {}): InProgressBackup {
|
|
155
|
+
return { id: 'rec-1', pid: 222, startedAt: new Date(NOW - 60_000), ...over };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
describe('resolveAbandonedBackups', () => {
|
|
159
|
+
// THE REGRESSION THIS EXISTS FOR (#616). Every earlier test supplied a
|
|
160
|
+
// staging directory, so the coupled implementation passed all of them while
|
|
161
|
+
// leaving 107 real records stranded. Nothing here mentions staging at all —
|
|
162
|
+
// that is the point.
|
|
163
|
+
test('resolves a dead record even though no staging directory exists', () => {
|
|
164
|
+
const d = recordDeps([record({ id: 'orphan', pid: 999 })], []);
|
|
165
|
+
const report = resolveAbandonedBackups(d);
|
|
166
|
+
|
|
167
|
+
expect(report.resolved).toEqual(['orphan']);
|
|
168
|
+
expect(d.failed).toEqual([{ id: 'orphan', message: ABANDONED_BACKUP_MESSAGE }]);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// All 107 rows on celilo-mgr predate the pid column. Age is the only signal
|
|
172
|
+
// they carry, so they must resolve on it rather than linger forever.
|
|
173
|
+
test('resolves a pid-less record once it is past the TTL', () => {
|
|
174
|
+
const d = recordDeps([
|
|
175
|
+
record({ id: 'pre-migration', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
expect(resolveAbandonedBackups(d).resolved).toEqual(['pre-migration']);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test('keeps a pid-less record that is still inside the TTL', () => {
|
|
182
|
+
const d = recordDeps([record({ id: 'recent', pid: null })]);
|
|
183
|
+
const report = resolveAbandonedBackups(d);
|
|
184
|
+
|
|
185
|
+
expect(report.resolved).toEqual([]);
|
|
186
|
+
expect(report.kept).toEqual(['recent']);
|
|
187
|
+
expect(d.failed).toEqual([]);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// Must never regress: a backup mid-flight is not abandoned.
|
|
191
|
+
test('KEEPS a record whose process is alive', () => {
|
|
192
|
+
const d = recordDeps([record({ id: 'live', pid: 222 })], [222]);
|
|
193
|
+
const report = resolveAbandonedBackups(d);
|
|
194
|
+
|
|
195
|
+
expect(report.resolved).toEqual([]);
|
|
196
|
+
expect(report.kept).toEqual(['live']);
|
|
197
|
+
expect(d.failed).toEqual([]);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test('resolves a record past the TTL even when its pid looks alive', () => {
|
|
201
|
+
// Same pid-reuse guard the reaper applies.
|
|
202
|
+
const d = recordDeps(
|
|
203
|
+
[record({ id: 'stale', pid: 222, startedAt: new Date(NOW - STAGING_TTL_MS - 1) })],
|
|
204
|
+
[222],
|
|
205
|
+
);
|
|
206
|
+
|
|
207
|
+
expect(resolveAbandonedBackups(d).resolved).toEqual(['stale']);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
test('sorts a mixed set without touching the live one', () => {
|
|
211
|
+
const d = recordDeps(
|
|
212
|
+
[
|
|
213
|
+
record({ id: 'dead', pid: 999 }),
|
|
214
|
+
record({ id: 'live', pid: 222 }),
|
|
215
|
+
record({ id: 'old', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
|
|
216
|
+
],
|
|
217
|
+
[222],
|
|
218
|
+
);
|
|
219
|
+
const report = resolveAbandonedBackups(d);
|
|
220
|
+
|
|
221
|
+
expect(report.resolved.sort()).toEqual(['dead', 'old']);
|
|
222
|
+
expect(report.kept).toEqual(['live']);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
test('nothing in progress is a no-op', () => {
|
|
226
|
+
const d = recordDeps([]);
|
|
227
|
+
const report = resolveAbandonedBackups(d);
|
|
228
|
+
|
|
229
|
+
expect(report).toEqual({ resolved: [], kept: [] });
|
|
230
|
+
expect(d.failed).toEqual([]);
|
|
231
|
+
});
|
|
232
|
+
});
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Cleaning up after backups whose process died — the staging they left on disk,
|
|
3
|
+
* and the records that still claim they are running.
|
|
4
|
+
*
|
|
5
|
+
* The two are separate obligations that share a predicate, NOT one obligation
|
|
6
|
+
* with two effects. Coupling them is exactly the bug in #616; see
|
|
7
|
+
* `resolveAbandonedBackups`.
|
|
3
8
|
*
|
|
4
9
|
* `backup-create.ts` assembles every envelope in a temp directory and removes
|
|
5
10
|
* it in a `finally`. That is correct and it is not enough: a `finally` does not
|
|
@@ -114,6 +119,73 @@ export interface ReapStagingReport {
|
|
|
114
119
|
ignored: string[];
|
|
115
120
|
}
|
|
116
121
|
|
|
122
|
+
/** An `in_progress` backup record, as seen by the record-resolution pass. */
|
|
123
|
+
export interface InProgressBackup {
|
|
124
|
+
id: string;
|
|
125
|
+
/** Null for records written before backups recorded their pid. */
|
|
126
|
+
pid: number | null;
|
|
127
|
+
startedAt: Date;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface ResolveAbandonedDeps {
|
|
131
|
+
listInProgress(): InProgressBackup[];
|
|
132
|
+
isPidRunnable(pid: number): boolean;
|
|
133
|
+
fail(recordId: string, message: string): void;
|
|
134
|
+
now(): number;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface ResolveAbandonedReport {
|
|
138
|
+
/** Records corrected from `in_progress` to failed. */
|
|
139
|
+
resolved: string[];
|
|
140
|
+
/** Left alone — a live backup owns them. */
|
|
141
|
+
kept: string[];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Correct records that still claim to be running after their process died.
|
|
146
|
+
*
|
|
147
|
+
* Deliberately INDEPENDENT of staging. The first version of this resolved
|
|
148
|
+
* records only as a side effect of reclaiming their staging directory, which
|
|
149
|
+
* was efficient — one liveness lookup serving two places — and strictly
|
|
150
|
+
* narrower than the requirement. A record whose staging is already gone was
|
|
151
|
+
* never visited, so it stayed `in_progress` forever: 107 such rows on
|
|
152
|
+
* celilo-mgr, the oldest from June, none of them reachable by the reaper
|
|
153
|
+
* because their directories had been cleared by hand (#616).
|
|
154
|
+
*
|
|
155
|
+
* That is not an edge case. `/tmp` is declared `D` in tmpfiles.d — cleared on
|
|
156
|
+
* boot — so ANY backup killed before a reboot loses its staging and becomes
|
|
157
|
+
* permanently unresolvable under the coupled design. Reclaiming disk and
|
|
158
|
+
* correcting records are two obligations that happen to share a predicate, not
|
|
159
|
+
* one obligation with two effects.
|
|
160
|
+
*
|
|
161
|
+
* The predicate itself is shared rather than reimplemented: this calls the same
|
|
162
|
+
* `reclaimReason` the reaper uses, so the TTL-versus-pid-reuse reasoning has
|
|
163
|
+
* exactly one home. A record is only ever resolved when its owner is provably
|
|
164
|
+
* gone; a live backup is left alone.
|
|
165
|
+
*/
|
|
166
|
+
export function resolveAbandonedBackups(deps: ResolveAbandonedDeps): ResolveAbandonedReport {
|
|
167
|
+
const report: ResolveAbandonedReport = { resolved: [], kept: [] };
|
|
168
|
+
|
|
169
|
+
for (const record of deps.listInProgress()) {
|
|
170
|
+
const reason = reclaimReason(
|
|
171
|
+
{ status: 'in_progress', pid: record.pid, startedAt: record.startedAt },
|
|
172
|
+
deps,
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
// `record-absent` and `record-terminal` are unreachable here — every row
|
|
176
|
+
// came from a query for in-progress records — so any reason at all means
|
|
177
|
+
// the owner is gone.
|
|
178
|
+
if (reason) {
|
|
179
|
+
deps.fail(record.id, ABANDONED_BACKUP_MESSAGE);
|
|
180
|
+
report.resolved.push(record.id);
|
|
181
|
+
} else {
|
|
182
|
+
report.kept.push(record.id);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return report;
|
|
187
|
+
}
|
|
188
|
+
|
|
117
189
|
/**
|
|
118
190
|
* Decide, for one staging directory, whether its owner is gone.
|
|
119
191
|
*
|
|
@@ -36,6 +36,7 @@ function deps(
|
|
|
36
36
|
pruned.push(id);
|
|
37
37
|
},
|
|
38
38
|
reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
|
|
39
|
+
resolveAbandonedRecords: () => ({ resolved: [], kept: [] }),
|
|
39
40
|
...overrides,
|
|
40
41
|
};
|
|
41
42
|
}
|
|
@@ -67,6 +68,20 @@ describe('runBackupSweep', () => {
|
|
|
67
68
|
expect(report.staging.reclaimed).toHaveLength(1);
|
|
68
69
|
});
|
|
69
70
|
|
|
71
|
+
// The sweep must resolve records whether or not the reaper found anything —
|
|
72
|
+
// that independence IS the fix for #616.
|
|
73
|
+
test('corrects abandoned records even when no staging was reclaimed', async () => {
|
|
74
|
+
const report = await runBackupSweep(
|
|
75
|
+
deps([moduleWith('forgejo', 'daily')], {
|
|
76
|
+
reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
|
|
77
|
+
resolveAbandonedRecords: () => ({ resolved: ['rec-a', 'rec-b'], kept: [] }),
|
|
78
|
+
}),
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
expect(report.staging.reclaimed).toEqual([]);
|
|
82
|
+
expect(report.records.resolved).toEqual(['rec-a', 'rec-b']);
|
|
83
|
+
});
|
|
84
|
+
|
|
70
85
|
test('reclaims staging even when no module is due to back up', async () => {
|
|
71
86
|
const report = await runBackupSweep(
|
|
72
87
|
deps([moduleWith('forgejo', 'daily')], {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
import type { ModuleManifest } from '../manifest/schema';
|
|
20
20
|
import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
|
|
21
|
-
import type { ReapStagingReport } from './backup-staging';
|
|
21
|
+
import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
|
|
22
22
|
import { InFlightError } from './module-operations';
|
|
23
23
|
|
|
24
24
|
export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
|
|
@@ -110,11 +110,21 @@ export interface BackupSweepDeps {
|
|
|
110
110
|
prune(module: BackupSweepModule): Promise<void>;
|
|
111
111
|
/** Reclaim staging left by backups whose process died. See backup-staging.ts. */
|
|
112
112
|
reapStaging(): ReapStagingReport;
|
|
113
|
+
/**
|
|
114
|
+
* Correct records that still claim to be running after their process died.
|
|
115
|
+
*
|
|
116
|
+
* Separate from `reapStaging` on purpose. Deriving this from what the reaper
|
|
117
|
+
* happened to reclaim left records stranded forever once their staging was
|
|
118
|
+
* gone — #616.
|
|
119
|
+
*/
|
|
120
|
+
resolveAbandonedRecords(): ResolveAbandonedReport;
|
|
113
121
|
}
|
|
114
122
|
|
|
115
123
|
export interface BackupSweepReport {
|
|
116
124
|
/** Staging reclaimed before this pass created any of its own. */
|
|
117
125
|
staging: ReapStagingReport;
|
|
126
|
+
/** Records corrected from a stale `in_progress`. */
|
|
127
|
+
records: ResolveAbandonedReport;
|
|
118
128
|
backedUp: string[];
|
|
119
129
|
/** Explicit `schedule: manual` — the author opted out. */
|
|
120
130
|
skippedManual: string[];
|
|
@@ -131,8 +141,14 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
|
|
|
131
141
|
// another ~4 GB of it, not once we are finished with it.
|
|
132
142
|
const staging = deps.reapStaging();
|
|
133
143
|
|
|
144
|
+
// Independent of the reap above, and that independence is the fix for #616:
|
|
145
|
+
// a record whose staging is already gone is invisible to the reaper, so
|
|
146
|
+
// resolving records off the reaper's results left 107 of them stranded.
|
|
147
|
+
const records = deps.resolveAbandonedRecords();
|
|
148
|
+
|
|
134
149
|
const report: BackupSweepReport = {
|
|
135
150
|
staging,
|
|
151
|
+
records,
|
|
136
152
|
backedUp: [],
|
|
137
153
|
skippedManual: [],
|
|
138
154
|
skippedNotDue: [],
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does `busInterview` actually PARK?
|
|
3
|
+
*
|
|
4
|
+
* celilo#609's whole scope rests on one claim: a command blocked on an
|
|
5
|
+
* unanswered interview stays alive indefinitely and resumes when the question
|
|
6
|
+
* is answered later. If true, #609 needs no command-state serialization — only
|
|
7
|
+
* session lifetime and re-attach. If the command instead dies quietly, #609 is
|
|
8
|
+
* a much larger design.
|
|
9
|
+
*
|
|
10
|
+
* That claim was originally read off the code (`timeoutMs: 0`), which is not
|
|
11
|
+
* the same as watching it happen. This file watches it happen: a real
|
|
12
|
+
* `module update` sweep parks on a breaking-update confirm nobody answers,
|
|
13
|
+
* stays parked, and then resumes and uses the answer when one arrives by event
|
|
14
|
+
* id — the same reply `celilo events reply <id> <value>` emits.
|
|
15
|
+
*
|
|
16
|
+
* The responder here answers `responder.probe` but deliberately ignores the
|
|
17
|
+
* interview, which is exactly the #609 situation: a responder exists (the
|
|
18
|
+
* api-serve bridge), so the fail-fast guard passes, but nobody can decide.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
|
22
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
23
|
+
import { tmpdir } from 'node:os';
|
|
24
|
+
import { join } from 'node:path';
|
|
25
|
+
import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bus';
|
|
26
|
+
import { handleModuleUpdate } from '../cli/commands/module-update';
|
|
27
|
+
import { getDb } from '../db/client';
|
|
28
|
+
import { modules } from '../db/schema';
|
|
29
|
+
import { RESPONDER_PROBE_EVENT } from './responder-probe';
|
|
30
|
+
|
|
31
|
+
const NO_SCHEMAS = defineEvents({});
|
|
32
|
+
const QUERY_TYPE = 'interview.required.module-upgrade:iptables.apply_breaking';
|
|
33
|
+
|
|
34
|
+
let dir: string;
|
|
35
|
+
let server: ReturnType<typeof Bun.serve>;
|
|
36
|
+
let registryUrl: string;
|
|
37
|
+
|
|
38
|
+
/** A responder that proves liveness but never answers the question. */
|
|
39
|
+
function probeOnlyResponder(busDbPath: string): { bus: Bus; close: () => void } {
|
|
40
|
+
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
41
|
+
const watch = bus.watch(RESPONDER_PROBE_EVENT, async (event) => {
|
|
42
|
+
if (event.replyFor !== null) return;
|
|
43
|
+
bus.emitRaw(
|
|
44
|
+
`${event.type}.reply`,
|
|
45
|
+
{ kind: 'daemon', emittedBy: 'park-test' },
|
|
46
|
+
{ replyFor: event.id, emittedBy: 'park-test' },
|
|
47
|
+
);
|
|
48
|
+
});
|
|
49
|
+
return {
|
|
50
|
+
bus,
|
|
51
|
+
close: () => {
|
|
52
|
+
watch.close();
|
|
53
|
+
bus.close();
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
59
|
+
|
|
60
|
+
beforeEach(() => {
|
|
61
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-park-'));
|
|
62
|
+
process.env.CELILO_DB_PATH = join(dir, 'test.db');
|
|
63
|
+
process.env.CELILO_ORIGINAL_CWD = dir;
|
|
64
|
+
process.env.EVENT_BUS_DB = join(dir, 'events.db');
|
|
65
|
+
|
|
66
|
+
getDb()
|
|
67
|
+
.insert(modules)
|
|
68
|
+
.values({
|
|
69
|
+
id: 'iptables',
|
|
70
|
+
name: 'iptables',
|
|
71
|
+
sourcePath: join(dir, 'installed'),
|
|
72
|
+
version: '1.0.2+9',
|
|
73
|
+
manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
|
|
74
|
+
})
|
|
75
|
+
.run();
|
|
76
|
+
|
|
77
|
+
server = Bun.serve({
|
|
78
|
+
port: 0,
|
|
79
|
+
fetch(req) {
|
|
80
|
+
if (new URL(req.url).pathname === '/index/ip/ta/iptables') {
|
|
81
|
+
return new Response(
|
|
82
|
+
`${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
return new Response('not found', { status: 404 });
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
registryUrl = `http://localhost:${server.port}`;
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
afterEach(() => {
|
|
92
|
+
server.stop(true);
|
|
93
|
+
rmSync(dir, { recursive: true, force: true });
|
|
94
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
95
|
+
process.env.CELILO_ORIGINAL_CWD = undefined;
|
|
96
|
+
process.env.EVENT_BUS_DB = undefined;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('a command parks on an unanswered interview, then resumes with an answer given later', async () => {
|
|
100
|
+
const responder = probeOnlyResponder(join(dir, 'events.db'));
|
|
101
|
+
const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
// Start the sweep but do NOT await it — it should block on the confirm.
|
|
105
|
+
let settled = false;
|
|
106
|
+
const sweep = handleModuleUpdate([], { registry: registryUrl }).then((r) => {
|
|
107
|
+
settled = true;
|
|
108
|
+
return r;
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
// 1. It parks. The sweep resolves in milliseconds if it does NOT park, so
|
|
112
|
+
// any interval well past the 250ms bus poll proves the point — no reason
|
|
113
|
+
// to hold a CI runner for seconds to say it.
|
|
114
|
+
await sleep(600);
|
|
115
|
+
expect(settled).toBe(false);
|
|
116
|
+
|
|
117
|
+
// 2. The question is on the bus, unanswered, and identifiable by event id.
|
|
118
|
+
const queries = observer.recentEvents({ type: QUERY_TYPE });
|
|
119
|
+
expect(queries.length).toBe(1);
|
|
120
|
+
const query = queries[0] as BusEvent;
|
|
121
|
+
const replies = observer.recentEvents({ type: `${QUERY_TYPE}.reply` });
|
|
122
|
+
expect(replies.length).toBe(0);
|
|
123
|
+
|
|
124
|
+
// 3. Answer it out-of-band, exactly as `celilo events reply <id> false` does.
|
|
125
|
+
observer.emitRaw(
|
|
126
|
+
`${QUERY_TYPE}.reply`,
|
|
127
|
+
{ value: false },
|
|
128
|
+
{ replyFor: query.id, emittedBy: 'claude-config-responder' },
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
// 4. It resumes AND uses the answer: `false` is a genuine decline, so the
|
|
132
|
+
// summary must say declined — not "NOT declined", which is what we'd see
|
|
133
|
+
// if it had failed rather than parked.
|
|
134
|
+
const result = await sweep;
|
|
135
|
+
const report = result.success ? (result.message ?? '') : (result.error ?? '');
|
|
136
|
+
expect(report).toContain('operator declined');
|
|
137
|
+
expect(report).not.toContain('NOT declined');
|
|
138
|
+
expect(result.success).toBe(true);
|
|
139
|
+
} finally {
|
|
140
|
+
observer.close();
|
|
141
|
+
responder.close();
|
|
142
|
+
}
|
|
143
|
+
}, 30_000);
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The instrument the original report reached for cannot see this question.
|
|
147
|
+
*
|
|
148
|
+
* `celilo events list-pending` is `bus.pendingDeliveries()` — it reads the
|
|
149
|
+
* `deliveries` table (subscriber fan-out), while an unanswered interview is a
|
|
150
|
+
* row in `events` awaiting a correlated reply. So "list-pending returned []"
|
|
151
|
+
* was never evidence about the interview either way. This pins that down so
|
|
152
|
+
* #609 builds its observability gate on something that can actually observe.
|
|
153
|
+
*/
|
|
154
|
+
test('events list-pending cannot see a parked interview — it reads a different table', async () => {
|
|
155
|
+
const responder = probeOnlyResponder(join(dir, 'events.db'));
|
|
156
|
+
const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
const sweep = handleModuleUpdate([], { registry: registryUrl });
|
|
160
|
+
await sleep(600);
|
|
161
|
+
|
|
162
|
+
// The question is genuinely there...
|
|
163
|
+
const queries = observer.recentEvents({ type: QUERY_TYPE });
|
|
164
|
+
expect(queries.length).toBe(1);
|
|
165
|
+
|
|
166
|
+
// ...and list-pending shows nothing, because it is looking elsewhere.
|
|
167
|
+
expect(observer.pendingDeliveries({ limit: 100 })).toHaveLength(0);
|
|
168
|
+
|
|
169
|
+
observer.emitRaw(
|
|
170
|
+
`${QUERY_TYPE}.reply`,
|
|
171
|
+
{ value: false },
|
|
172
|
+
{ replyFor: queries[0].id, emittedBy: 'park-test' },
|
|
173
|
+
);
|
|
174
|
+
await sweep;
|
|
175
|
+
} finally {
|
|
176
|
+
observer.close();
|
|
177
|
+
responder.close();
|
|
178
|
+
}
|
|
179
|
+
}, 30_000);
|
|
@@ -7,14 +7,16 @@
|
|
|
7
7
|
*
|
|
8
8
|
* No timeouts: the deploy waits indefinitely for a responder. If
|
|
9
9
|
* nothing answers, the operator sees the unanswered query via
|
|
10
|
-
* `celilo events list-
|
|
10
|
+
* `celilo events list-unanswered` and answers it with `celilo events
|
|
11
|
+
* reply <id> <value>`. (NOT `events list-pending` — that reads the
|
|
12
|
+
* subscriber `deliveries` table and cannot see an unanswered query.)
|
|
11
13
|
*
|
|
12
14
|
* See `infra/openspec/changes/interactive-deploys-via-event-bus/proposal.md`.
|
|
13
15
|
*/
|
|
14
16
|
|
|
15
17
|
import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
|
|
16
18
|
import { getEventBusPath } from '../config/paths';
|
|
17
|
-
import {
|
|
19
|
+
import { InterviewAbandonedError } from './interview-errors';
|
|
18
20
|
import { ensureResponderForInterview } from './responder-probe';
|
|
19
21
|
|
|
20
22
|
const NO_SCHEMAS = defineEvents({});
|
|
@@ -241,12 +243,15 @@ export interface InterviewRequiredPayload {
|
|
|
241
243
|
export interface InterviewReply {
|
|
242
244
|
value: unknown;
|
|
243
245
|
/**
|
|
244
|
-
* Set instead of `value` when the
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
246
|
+
* Set instead of `value` when the question was reaped rather than decided —
|
|
247
|
+
* a parked session passed its TTL with nobody answering. `askInterview` turns
|
|
248
|
+
* it into an `InterviewAbandonedError`, which is distinct from both a decline
|
|
249
|
+
* (someone said no) and an unanswered question (still standing).
|
|
250
|
+
*
|
|
251
|
+
* A responder that merely *cannot* decide emits nothing at all: the query
|
|
252
|
+
* stays unanswered and the asking command stays parked (celilo#609).
|
|
248
253
|
*/
|
|
249
|
-
|
|
254
|
+
abandoned?: { reason: string };
|
|
250
255
|
}
|
|
251
256
|
|
|
252
257
|
/**
|
|
@@ -324,7 +329,7 @@ export async function askInterview(
|
|
|
324
329
|
): Promise<unknown> {
|
|
325
330
|
const type = EVENT_TYPES.interviewRequired(payload.scope, payload.key);
|
|
326
331
|
const reply = await busInterviewGuarded<InterviewReply>(type, payload, ownerBus);
|
|
327
|
-
if (reply.
|
|
332
|
+
if (reply.abandoned) throw new InterviewAbandonedError(type, reply.abandoned.reason);
|
|
328
333
|
return reply.value;
|
|
329
334
|
}
|
|
330
335
|
|
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* The two error types that mean "this interview question was not decided".
|
|
3
|
+
* Their own module so `responder-probe` (nobody is listening at all) and the
|
|
4
|
+
* session reaper (a parked question expired) can both throw without importing
|
|
5
|
+
* each other.
|
|
6
6
|
*
|
|
7
|
-
* Callers catch
|
|
8
|
-
*
|
|
9
|
-
* update as "operator declined" that no operator had ever seen.
|
|
7
|
+
* Callers catch these to distinguish a question that was never decided from one
|
|
8
|
+
* that was answered — the distinction `module update` conflated when it reported
|
|
9
|
+
* a breaking update as "operator declined" that no operator had ever seen. The
|
|
10
|
+
* two are not interchangeable: *unanswered* means no responder could even be
|
|
11
|
+
* found, *abandoned* means a responder existed, the question stood, and the
|
|
12
|
+
* deadline passed with nobody deciding.
|
|
10
13
|
*/
|
|
11
14
|
export class InterviewUnansweredError extends Error {
|
|
12
15
|
constructor(
|
|
@@ -18,3 +21,17 @@ export class InterviewUnansweredError extends Error {
|
|
|
18
21
|
this.name = 'InterviewUnansweredError';
|
|
19
22
|
}
|
|
20
23
|
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The question was posted, stood unanswered past its session's TTL, and the
|
|
27
|
+
* reaper answered it `abandoned` to release what the parked command held.
|
|
28
|
+
*/
|
|
29
|
+
export class InterviewAbandonedError extends Error {
|
|
30
|
+
constructor(
|
|
31
|
+
readonly queryType: string,
|
|
32
|
+
message: string,
|
|
33
|
+
) {
|
|
34
|
+
super(message);
|
|
35
|
+
this.name = 'InterviewAbandonedError';
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -78,12 +78,18 @@ test('answers responder.probe with kind daemon', async () => {
|
|
|
78
78
|
}, 15_000);
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
81
|
+
* A responder that cannot reach a decider must emit NOTHING (celilo#609).
|
|
82
|
+
*
|
|
83
|
+
* PR #607 had it reply `{error}` here, which reads as "fail loudly" but carries
|
|
84
|
+
* `replyFor: <query id>` — it *consumes the query*. The question then no longer
|
|
85
|
+
* exists for anyone else to answer, and the command dies with it. Leaving it
|
|
86
|
+
* unanswered parks the command instead, so a later responder can still decide.
|
|
87
|
+
*
|
|
88
|
+
* Asserted on the value that comes back, not on liveness: the reply emitted
|
|
89
|
+
* afterwards correlates to the original query, which is only true if the
|
|
90
|
+
* responder left it standing.
|
|
85
91
|
*/
|
|
86
|
-
test('an ask that cannot be answered
|
|
92
|
+
test('an ask that cannot be answered emits no reply — the query stays answerable', async () => {
|
|
87
93
|
const responder = startRemoteResponder({
|
|
88
94
|
busDbPath,
|
|
89
95
|
ask: async () => {
|
|
@@ -92,24 +98,31 @@ test('an ask that cannot be answered replies with an error, not the default', as
|
|
|
92
98
|
});
|
|
93
99
|
|
|
94
100
|
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
101
|
+
const type = 'interview.required.module-upgrade:iptables.apply_breaking';
|
|
95
102
|
try {
|
|
96
|
-
const
|
|
97
|
-
'
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
))
|
|
103
|
+
const query = bus.emitRaw(type, {
|
|
104
|
+
scope: 'module-upgrade:iptables',
|
|
105
|
+
key: 'apply_breaking',
|
|
106
|
+
kind: 'confirm',
|
|
107
|
+
message: 'Apply breaking update for iptables?',
|
|
108
|
+
required: true,
|
|
109
|
+
defaultValue: 'false',
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
// Well past the responder's watch latency: still nobody has answered.
|
|
113
|
+
await new Promise((r) => setTimeout(r, 600));
|
|
114
|
+
expect(bus.recentEvents({ type: `${type}.reply` })).toHaveLength(0);
|
|
108
115
|
|
|
116
|
+
// And the query is still live: a reply emitted now is a genuine answer.
|
|
117
|
+
bus.emitRaw(
|
|
118
|
+
`${type}.reply`,
|
|
119
|
+
{ value: false },
|
|
120
|
+
{ replyFor: query.id, emittedBy: 'later-responder' },
|
|
121
|
+
);
|
|
122
|
+
const replies = bus.recentEvents({ type: `${type}.reply` });
|
|
109
123
|
expect(replies).toHaveLength(1);
|
|
110
|
-
|
|
111
|
-
expect(
|
|
112
|
-
expect(payload.error).toContain('terminal');
|
|
124
|
+
expect((replies[0].payload as { value: unknown }).value).toBe(false);
|
|
125
|
+
expect(replies[0].replyFor).toBe(query.id);
|
|
113
126
|
} finally {
|
|
114
127
|
bus.close();
|
|
115
128
|
responder.close();
|
|
@@ -72,12 +72,16 @@ export function startRemoteResponder(opts: RemoteResponderOptions): RemoteRespon
|
|
|
72
72
|
required: payload.required,
|
|
73
73
|
});
|
|
74
74
|
} catch (err) {
|
|
75
|
-
// The client had no way to answer
|
|
76
|
-
//
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
75
|
+
// The client had no way to answer — so we emit NOTHING. A reply of any
|
|
76
|
+
// shape consumes the query (it carries `replyFor: event.id`), destroying
|
|
77
|
+
// a question nobody has answered yet; the asking command then dies with
|
|
78
|
+
// it and no other responder can ever act. Leaving it unanswered parks the
|
|
79
|
+
// command instead, which is what `busInterview`'s `timeoutMs: 0` is for.
|
|
80
|
+
// The client learns it is parked from the `blocked` wire message.
|
|
81
|
+
process.stderr.write(
|
|
82
|
+
`[remote-responder] parked ${event.type} (#${event.id}): ${
|
|
83
|
+
err instanceof Error ? err.message : String(err)
|
|
84
|
+
}\n`,
|
|
81
85
|
);
|
|
82
86
|
return;
|
|
83
87
|
}
|