@celilo/cli 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/CELILO_SUBSYSTEMS.md +4 -2
  2. package/package.json +4 -4
  3. package/src/api/remote-client.test.ts +86 -2
  4. package/src/api/serve.ts +242 -38
  5. package/src/api/sessions.test.ts +196 -0
  6. package/src/api/sessions.ts +278 -0
  7. package/src/cli/commands/apt-upgrade.test.ts +20 -1
  8. package/src/cli/commands/apt-upgrade.ts +12 -2
  9. package/src/cli/commands/backup-sweep.ts +25 -9
  10. package/src/cli/commands/events.ts +150 -4
  11. package/src/cli/commands/module-update.test.ts +72 -1
  12. package/src/cli/commands/module-update.ts +68 -22
  13. package/src/cli/commands/system-migrate.test.ts +56 -0
  14. package/src/cli/commands/system-migrate.ts +52 -4
  15. package/src/cli/completion.ts +2 -0
  16. package/src/cli/index.ts +27 -3
  17. package/src/db/migration-status.test.ts +114 -0
  18. package/src/db/migration-status.ts +78 -0
  19. package/src/db/schema-introspection.ts +8 -1
  20. package/src/services/backup-metadata.ts +17 -0
  21. package/src/services/backup-staging.test.ts +98 -0
  22. package/src/services/backup-staging.ts +73 -1
  23. package/src/services/backup-sweep.test.ts +15 -0
  24. package/src/services/backup-sweep.ts +17 -1
  25. package/src/services/bus-interview-park.test.ts +179 -0
  26. package/src/services/bus-interview.ts +17 -6
  27. package/src/services/events-daemon.test.ts +244 -0
  28. package/src/services/events-daemon.ts +295 -8
  29. package/src/services/fleet-checks.test.ts +75 -4
  30. package/src/services/fleet-checks.ts +82 -12
  31. package/src/services/interview-errors.ts +37 -0
  32. package/src/services/remote-responder.test.ts +83 -0
  33. package/src/services/remote-responder.ts +31 -10
  34. package/src/services/responder-probe.ts +3 -1
@@ -0,0 +1,114 @@
1
+ /**
2
+ * celilo#604: a table COUNT reported "up to date" for a DB missing a migrated
3
+ * COLUMN, so the rollout's own stop-condition was uncheckable by the tools the
4
+ * runbook named. These assert the column case specifically.
5
+ */
6
+
7
+ import { Database } from 'bun:sqlite';
8
+ import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
9
+ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
10
+ import { tmpdir } from 'node:os';
11
+ import { join } from 'node:path';
12
+ import { getMigrationStatus, readMigrationJournal } from './migration-status';
13
+
14
+ /** A journal + a `__drizzle_migrations` table, wired the way drizzle wires them. */
15
+ function seed(dir: string, entries: Array<{ when: number; tag: string }>, appliedWhens: number[]) {
16
+ mkdirSync(join(dir, 'meta'), { recursive: true });
17
+ writeFileSync(
18
+ join(dir, 'meta', '_journal.json'),
19
+ JSON.stringify({ version: '7', dialect: 'sqlite', entries }),
20
+ );
21
+ const db = new Database(':memory:');
22
+ db.run(
23
+ 'CREATE TABLE `__drizzle_migrations` (id INTEGER PRIMARY KEY, hash TEXT, created_at NUMERIC)',
24
+ );
25
+ for (const when of appliedWhens) {
26
+ db.run('INSERT INTO `__drizzle_migrations` (hash, created_at) VALUES (?, ?)', [
27
+ `h${when}`,
28
+ when,
29
+ ]);
30
+ }
31
+ return db;
32
+ }
33
+
34
+ describe('getMigrationStatus', () => {
35
+ let dir: string;
36
+ beforeEach(() => {
37
+ dir = mkdtempSync(join(tmpdir(), 'celilo-migstatus-'));
38
+ });
39
+ afterEach(() => rmSync(dir, { recursive: true, force: true }));
40
+
41
+ it('names the latest applied migration and every pending one', () => {
42
+ const db = seed(
43
+ dir,
44
+ [
45
+ { when: 100, tag: '0018_drop_alert_policy_snapshot' },
46
+ { when: 200, tag: '0019_backup_pid' },
47
+ { when: 300, tag: '0020_future' },
48
+ ],
49
+ [100, 200],
50
+ );
51
+
52
+ const status = getMigrationStatus(db, dir);
53
+
54
+ expect(status.appliedCount).toBe(2);
55
+ expect(status.latestApplied).toBe('0019_backup_pid');
56
+ expect(status.pending).toEqual(['0020_future']);
57
+ });
58
+
59
+ it('reports nothing pending once every journal entry is applied', () => {
60
+ const db = seed(dir, [{ when: 100, tag: '0000_init' }], [100]);
61
+ const status = getMigrationStatus(db, dir);
62
+ expect(status.pending).toEqual([]);
63
+ expect(status.latestApplied).toBe('0000_init');
64
+ });
65
+
66
+ it('reports a DB that has never been migrated as all-pending', () => {
67
+ mkdirSync(join(dir, 'meta'), { recursive: true });
68
+ writeFileSync(
69
+ join(dir, 'meta', '_journal.json'),
70
+ JSON.stringify({ entries: [{ when: 1, tag: '0000_init' }] }),
71
+ );
72
+ const status = getMigrationStatus(new Database(':memory:'), dir);
73
+ expect(status.appliedCount).toBe(0);
74
+ expect(status.latestApplied).toBeNull();
75
+ expect(status.pending).toEqual(['0000_init']);
76
+ });
77
+
78
+ it('counts columns, not just tables — the signal a column migration needs', () => {
79
+ const db = seed(dir, [], []);
80
+ const status = getMigrationStatus(db, dir);
81
+ // Every schema table is missing here, so the counts are the code's totals.
82
+ expect(status.tableCount).toBeGreaterThan(0);
83
+ expect(status.columnCount).toBeGreaterThan(status.tableCount);
84
+ });
85
+
86
+ it('reads journal entries oldest-first regardless of file order', () => {
87
+ mkdirSync(join(dir, 'meta'), { recursive: true });
88
+ writeFileSync(
89
+ join(dir, 'meta', '_journal.json'),
90
+ JSON.stringify({
91
+ entries: [
92
+ { when: 300, tag: 'c' },
93
+ { when: 100, tag: 'a' },
94
+ ],
95
+ }),
96
+ );
97
+ expect(readMigrationJournal(dir).map((e) => e.tag)).toEqual(['a', 'c']);
98
+ });
99
+
100
+ it('returns an empty journal when the folder has none', () => {
101
+ expect(readMigrationJournal(join(dir, 'nope'))).toEqual([]);
102
+ });
103
+ });
104
+
105
+ describe('getMigrationStatus against the real migrations folder', () => {
106
+ it('sees backups.pid as missing when the column migration did not land', () => {
107
+ // The exact celilo#604 shape: every table present, one migrated column not.
108
+ const db = new Database(':memory:');
109
+ db.run('CREATE TABLE backups (id TEXT PRIMARY KEY)');
110
+ const status = getMigrationStatus(db, join(import.meta.dir, '../../drizzle'));
111
+ expect(status.missingColumns).toContain('backups.pid');
112
+ expect(status.pending).toContain('0019_backup_pid');
113
+ });
114
+ });
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Read-only migration interrogation (celilo#604).
3
+ *
4
+ * `system migrate` used to report "Schema current: 35 tables" — a table COUNT,
5
+ * which cannot distinguish "the column migration applied" from "nothing
6
+ * happened". 0019_backup_pid adds a column; a rollout runbook asserting
7
+ * "applied 19 → 20, backups.pid present" had no product surface to check it
8
+ * against and had to reach for `sqlite3` over SSH.
9
+ *
10
+ * This names migrations. Drizzle's `__drizzle_migrations.created_at` is the
11
+ * journal entry's `when`, so the join back to a human tag is exact.
12
+ */
13
+
14
+ import type { Database } from 'bun:sqlite';
15
+ import { existsSync, readFileSync } from 'node:fs';
16
+ import { join } from 'node:path';
17
+ import { findSchemaDrift } from './schema-introspection';
18
+
19
+ export interface MigrationStatus {
20
+ /** Rows in `__drizzle_migrations` — what the runbook calls "applied count". */
21
+ appliedCount: number;
22
+ /** Tag of the newest applied migration, e.g. `0019_backup_pid`. */
23
+ latestApplied: string | null;
24
+ /** Tags present on disk that this DB has not applied, oldest first. */
25
+ pending: string[];
26
+ /** Tables the running code declares that the DB lacks. */
27
+ missingTables: string[];
28
+ /** `table.column` the running code declares that the DB lacks. */
29
+ missingColumns: string[];
30
+ tableCount: number;
31
+ columnCount: number;
32
+ }
33
+
34
+ interface JournalEntry {
35
+ when: number;
36
+ tag: string;
37
+ }
38
+
39
+ /** Journal entries (oldest first), or [] when the folder has no journal. */
40
+ export function readMigrationJournal(migrationsFolder: string): JournalEntry[] {
41
+ const path = join(migrationsFolder, 'meta', '_journal.json');
42
+ if (!existsSync(path)) return [];
43
+ const parsed = JSON.parse(readFileSync(path, 'utf-8')) as { entries?: JournalEntry[] };
44
+ return [...(parsed.entries ?? [])].sort((a, b) => a.when - b.when);
45
+ }
46
+
47
+ /** `created_at` timestamps recorded in `__drizzle_migrations` (empty if untracked). */
48
+ function appliedTimestamps(sqlite: Database): number[] {
49
+ try {
50
+ return sqlite
51
+ .query<{ created_at: number }, []>(
52
+ 'SELECT created_at FROM `__drizzle_migrations` ORDER BY created_at',
53
+ )
54
+ .all()
55
+ .map((r) => r.created_at);
56
+ } catch {
57
+ // No migrations table — a DB that has never been through the migrator.
58
+ return [];
59
+ }
60
+ }
61
+
62
+ export function getMigrationStatus(sqlite: Database, migrationsFolder: string): MigrationStatus {
63
+ const journal = readMigrationJournal(migrationsFolder);
64
+ const applied = new Set(appliedTimestamps(sqlite));
65
+ const appliedTags = journal.filter((e) => applied.has(e.when)).map((e) => e.tag);
66
+ const pending = journal.filter((e) => !applied.has(e.when)).map((e) => e.tag);
67
+ const drift = findSchemaDrift(sqlite);
68
+
69
+ return {
70
+ appliedCount: applied.size,
71
+ latestApplied: appliedTags.at(-1) ?? null,
72
+ pending,
73
+ missingTables: drift.missingTables,
74
+ missingColumns: drift.missingColumns,
75
+ tableCount: drift.tableCount,
76
+ columnCount: drift.columnCount,
77
+ };
78
+ }
@@ -20,6 +20,12 @@ export interface SchemaDrift {
20
20
  missingColumns: string[];
21
21
  /** Total number of tables the code's schema declares. */
22
22
  tableCount: number;
23
+ /**
24
+ * Total number of columns the code's schema declares. Reported alongside
25
+ * tableCount so the doctor can say what it actually checked — a table count
26
+ * alone reads as "columns unverified" even when they were (celilo#604).
27
+ */
28
+ columnCount: number;
23
29
  }
24
30
 
25
31
  /** Every table name + column names the drizzle schema declares. */
@@ -84,5 +90,6 @@ export function findSchemaDrift(sqlite: Database): SchemaDrift {
84
90
  if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
85
91
  }
86
92
  }
87
- return { missingTables, missingColumns, tableCount: tables.length };
93
+ const columnCount = tables.reduce((n, t) => n + t.columns.length, 0);
94
+ return { missingTables, missingColumns, tableCount: tables.length, columnCount };
88
95
  }
@@ -156,6 +156,23 @@ export function listBackups(options?: {
156
156
  return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
157
157
  }
158
158
 
159
+ /**
160
+ * Every backup record still claiming to be in progress, oldest first.
161
+ *
162
+ * Feeds the record-resolution pass (`resolveAbandonedBackups`), which corrects
163
+ * the ones whose process is gone. Unbounded on purpose — there is no sensible
164
+ * limit on "how many lies to fix", and celilo-mgr had 107 of them.
165
+ */
166
+ export function listInProgressBackups(): Backup[] {
167
+ const db = getDb();
168
+ return db
169
+ .select()
170
+ .from(backups)
171
+ .where(eq(backups.status, 'in_progress'))
172
+ .orderBy(backups.startedAt)
173
+ .all();
174
+ }
175
+
159
176
  /**
160
177
  * List completed backups for a specific module, ordered newest first
161
178
  */
@@ -1,10 +1,14 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
2
  import {
3
+ ABANDONED_BACKUP_MESSAGE,
4
+ type InProgressBackup,
3
5
  type ReapStagingDeps,
6
+ type ResolveAbandonedDeps,
4
7
  STAGING_PREFIX,
5
8
  STAGING_TTL_MS,
6
9
  type StagingOwner,
7
10
  reapOrphanedStaging,
11
+ resolveAbandonedBackups,
8
12
  stagingDirFor,
9
13
  } from './backup-staging';
10
14
 
@@ -132,3 +136,97 @@ describe('reapOrphanedStaging', () => {
132
136
  expect(report.kept).toEqual([locked]);
133
137
  });
134
138
  });
139
+
140
+ function recordDeps(
141
+ records: InProgressBackup[],
142
+ runnablePids: number[] = [],
143
+ ): ResolveAbandonedDeps & { failed: Array<{ id: string; message: string }> } {
144
+ const failed: Array<{ id: string; message: string }> = [];
145
+ return {
146
+ failed,
147
+ listInProgress: () => records,
148
+ isPidRunnable: (pid) => runnablePids.includes(pid),
149
+ fail: (id, message) => failed.push({ id, message }),
150
+ now: () => NOW,
151
+ };
152
+ }
153
+
154
+ function record(over: Partial<InProgressBackup> = {}): InProgressBackup {
155
+ return { id: 'rec-1', pid: 222, startedAt: new Date(NOW - 60_000), ...over };
156
+ }
157
+
158
+ describe('resolveAbandonedBackups', () => {
159
+ // THE REGRESSION THIS EXISTS FOR (#616). Every earlier test supplied a
160
+ // staging directory, so the coupled implementation passed all of them while
161
+ // leaving 107 real records stranded. Nothing here mentions staging at all —
162
+ // that is the point.
163
+ test('resolves a dead record even though no staging directory exists', () => {
164
+ const d = recordDeps([record({ id: 'orphan', pid: 999 })], []);
165
+ const report = resolveAbandonedBackups(d);
166
+
167
+ expect(report.resolved).toEqual(['orphan']);
168
+ expect(d.failed).toEqual([{ id: 'orphan', message: ABANDONED_BACKUP_MESSAGE }]);
169
+ });
170
+
171
+ // All 107 rows on celilo-mgr predate the pid column. Age is the only signal
172
+ // they carry, so they must resolve on it rather than linger forever.
173
+ test('resolves a pid-less record once it is past the TTL', () => {
174
+ const d = recordDeps([
175
+ record({ id: 'pre-migration', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
176
+ ]);
177
+
178
+ expect(resolveAbandonedBackups(d).resolved).toEqual(['pre-migration']);
179
+ });
180
+
181
+ test('keeps a pid-less record that is still inside the TTL', () => {
182
+ const d = recordDeps([record({ id: 'recent', pid: null })]);
183
+ const report = resolveAbandonedBackups(d);
184
+
185
+ expect(report.resolved).toEqual([]);
186
+ expect(report.kept).toEqual(['recent']);
187
+ expect(d.failed).toEqual([]);
188
+ });
189
+
190
+ // Must never regress: a backup mid-flight is not abandoned.
191
+ test('KEEPS a record whose process is alive', () => {
192
+ const d = recordDeps([record({ id: 'live', pid: 222 })], [222]);
193
+ const report = resolveAbandonedBackups(d);
194
+
195
+ expect(report.resolved).toEqual([]);
196
+ expect(report.kept).toEqual(['live']);
197
+ expect(d.failed).toEqual([]);
198
+ });
199
+
200
+ test('resolves a record past the TTL even when its pid looks alive', () => {
201
+ // Same pid-reuse guard the reaper applies.
202
+ const d = recordDeps(
203
+ [record({ id: 'stale', pid: 222, startedAt: new Date(NOW - STAGING_TTL_MS - 1) })],
204
+ [222],
205
+ );
206
+
207
+ expect(resolveAbandonedBackups(d).resolved).toEqual(['stale']);
208
+ });
209
+
210
+ test('sorts a mixed set without touching the live one', () => {
211
+ const d = recordDeps(
212
+ [
213
+ record({ id: 'dead', pid: 999 }),
214
+ record({ id: 'live', pid: 222 }),
215
+ record({ id: 'old', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
216
+ ],
217
+ [222],
218
+ );
219
+ const report = resolveAbandonedBackups(d);
220
+
221
+ expect(report.resolved.sort()).toEqual(['dead', 'old']);
222
+ expect(report.kept).toEqual(['live']);
223
+ });
224
+
225
+ test('nothing in progress is a no-op', () => {
226
+ const d = recordDeps([]);
227
+ const report = resolveAbandonedBackups(d);
228
+
229
+ expect(report).toEqual({ resolved: [], kept: [] });
230
+ expect(d.failed).toEqual([]);
231
+ });
232
+ });
@@ -1,5 +1,10 @@
1
1
  /**
2
- * Reclaiming backup staging directories whose owner is gone.
2
+ * Cleaning up after backups whose process died — the staging they left on disk,
3
+ * and the records that still claim they are running.
4
+ *
5
+ * The two are separate obligations that share a predicate, NOT one obligation
6
+ * with two effects. Coupling them is exactly the bug in #616; see
7
+ * `resolveAbandonedBackups`.
3
8
  *
4
9
  * `backup-create.ts` assembles every envelope in a temp directory and removes
5
10
  * it in a `finally`. That is correct and it is not enough: a `finally` does not
@@ -114,6 +119,73 @@ export interface ReapStagingReport {
114
119
  ignored: string[];
115
120
  }
116
121
 
122
+ /** An `in_progress` backup record, as seen by the record-resolution pass. */
123
+ export interface InProgressBackup {
124
+ id: string;
125
+ /** Null for records written before backups recorded their pid. */
126
+ pid: number | null;
127
+ startedAt: Date;
128
+ }
129
+
130
+ export interface ResolveAbandonedDeps {
131
+ listInProgress(): InProgressBackup[];
132
+ isPidRunnable(pid: number): boolean;
133
+ fail(recordId: string, message: string): void;
134
+ now(): number;
135
+ }
136
+
137
+ export interface ResolveAbandonedReport {
138
+ /** Records corrected from `in_progress` to failed. */
139
+ resolved: string[];
140
+ /** Left alone — a live backup owns them. */
141
+ kept: string[];
142
+ }
143
+
144
+ /**
145
+ * Correct records that still claim to be running after their process died.
146
+ *
147
+ * Deliberately INDEPENDENT of staging. The first version of this resolved
148
+ * records only as a side effect of reclaiming their staging directory, which
149
+ * was efficient — one liveness lookup serving two places — and strictly
150
+ * narrower than the requirement. A record whose staging is already gone was
151
+ * never visited, so it stayed `in_progress` forever: 107 such rows on
152
+ * celilo-mgr, the oldest from June, none of them reachable by the reaper
153
+ * because their directories had been cleared by hand (#616).
154
+ *
155
+ * That is not an edge case. `/tmp` is declared `D` in tmpfiles.d — cleared on
156
+ * boot — so ANY backup killed before a reboot loses its staging and becomes
157
+ * permanently unresolvable under the coupled design. Reclaiming disk and
158
+ * correcting records are two obligations that happen to share a predicate, not
159
+ * one obligation with two effects.
160
+ *
161
+ * The predicate itself is shared rather than reimplemented: this calls the same
162
+ * `reclaimReason` the reaper uses, so the TTL-versus-pid-reuse reasoning has
163
+ * exactly one home. A record is only ever resolved when its owner is provably
164
+ * gone; a live backup is left alone.
165
+ */
166
+ export function resolveAbandonedBackups(deps: ResolveAbandonedDeps): ResolveAbandonedReport {
167
+ const report: ResolveAbandonedReport = { resolved: [], kept: [] };
168
+
169
+ for (const record of deps.listInProgress()) {
170
+ const reason = reclaimReason(
171
+ { status: 'in_progress', pid: record.pid, startedAt: record.startedAt },
172
+ deps,
173
+ );
174
+
175
+ // `record-absent` and `record-terminal` are unreachable here — every row
176
+ // came from a query for in-progress records — so any reason at all means
177
+ // the owner is gone.
178
+ if (reason) {
179
+ deps.fail(record.id, ABANDONED_BACKUP_MESSAGE);
180
+ report.resolved.push(record.id);
181
+ } else {
182
+ report.kept.push(record.id);
183
+ }
184
+ }
185
+
186
+ return report;
187
+ }
188
+
117
189
  /**
118
190
  * Decide, for one staging directory, whether its owner is gone.
119
191
  *
@@ -36,6 +36,7 @@ function deps(
36
36
  pruned.push(id);
37
37
  },
38
38
  reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
39
+ resolveAbandonedRecords: () => ({ resolved: [], kept: [] }),
39
40
  ...overrides,
40
41
  };
41
42
  }
@@ -67,6 +68,20 @@ describe('runBackupSweep', () => {
67
68
  expect(report.staging.reclaimed).toHaveLength(1);
68
69
  });
69
70
 
71
+ // The sweep must resolve records whether or not the reaper found anything —
72
+ // that independence IS the fix for #616.
73
+ test('corrects abandoned records even when no staging was reclaimed', async () => {
74
+ const report = await runBackupSweep(
75
+ deps([moduleWith('forgejo', 'daily')], {
76
+ reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
77
+ resolveAbandonedRecords: () => ({ resolved: ['rec-a', 'rec-b'], kept: [] }),
78
+ }),
79
+ );
80
+
81
+ expect(report.staging.reclaimed).toEqual([]);
82
+ expect(report.records.resolved).toEqual(['rec-a', 'rec-b']);
83
+ });
84
+
70
85
  test('reclaims staging even when no module is due to back up', async () => {
71
86
  const report = await runBackupSweep(
72
87
  deps([moduleWith('forgejo', 'daily')], {
@@ -18,7 +18,7 @@
18
18
 
19
19
  import type { ModuleManifest } from '../manifest/schema';
20
20
  import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
21
- import type { ReapStagingReport } from './backup-staging';
21
+ import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
22
22
  import { InFlightError } from './module-operations';
23
23
 
24
24
  export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
@@ -110,11 +110,21 @@ export interface BackupSweepDeps {
110
110
  prune(module: BackupSweepModule): Promise<void>;
111
111
  /** Reclaim staging left by backups whose process died. See backup-staging.ts. */
112
112
  reapStaging(): ReapStagingReport;
113
+ /**
114
+ * Correct records that still claim to be running after their process died.
115
+ *
116
+ * Separate from `reapStaging` on purpose. Deriving this from what the reaper
117
+ * happened to reclaim left records stranded forever once their staging was
118
+ * gone — #616.
119
+ */
120
+ resolveAbandonedRecords(): ResolveAbandonedReport;
113
121
  }
114
122
 
115
123
  export interface BackupSweepReport {
116
124
  /** Staging reclaimed before this pass created any of its own. */
117
125
  staging: ReapStagingReport;
126
+ /** Records corrected from a stale `in_progress`. */
127
+ records: ResolveAbandonedReport;
118
128
  backedUp: string[];
119
129
  /** Explicit `schedule: manual` — the author opted out. */
120
130
  skippedManual: string[];
@@ -131,8 +141,14 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
131
141
  // another ~4 GB of it, not once we are finished with it.
132
142
  const staging = deps.reapStaging();
133
143
 
144
+ // Independent of the reap above, and that independence is the fix for #616:
145
+ // a record whose staging is already gone is invisible to the reaper, so
146
+ // resolving records off the reaper's results left 107 of them stranded.
147
+ const records = deps.resolveAbandonedRecords();
148
+
134
149
  const report: BackupSweepReport = {
135
150
  staging,
151
+ records,
136
152
  backedUp: [],
137
153
  skippedManual: [],
138
154
  skippedNotDue: [],
@@ -0,0 +1,179 @@
1
+ /**
2
+ * Does `busInterview` actually PARK?
3
+ *
4
+ * celilo#609's whole scope rests on one claim: a command blocked on an
5
+ * unanswered interview stays alive indefinitely and resumes when the question
6
+ * is answered later. If true, #609 needs no command-state serialization — only
7
+ * session lifetime and re-attach. If the command instead dies quietly, #609 is
8
+ * a much larger design.
9
+ *
10
+ * That claim was originally read off the code (`timeoutMs: 0`), which is not
11
+ * the same as watching it happen. This file watches it happen: a real
12
+ * `module update` sweep parks on a breaking-update confirm nobody answers,
13
+ * stays parked, and then resumes and uses the answer when one arrives by event
14
+ * id — the same reply `celilo events reply <id> <value>` emits.
15
+ *
16
+ * The responder here answers `responder.probe` but deliberately ignores the
17
+ * interview, which is exactly the #609 situation: a responder exists (the
18
+ * api-serve bridge), so the fail-fast guard passes, but nobody can decide.
19
+ */
20
+
21
+ import { afterEach, beforeEach, expect, test } from 'bun:test';
22
+ import { mkdtempSync, rmSync } from 'node:fs';
23
+ import { tmpdir } from 'node:os';
24
+ import { join } from 'node:path';
25
+ import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bus';
26
+ import { handleModuleUpdate } from '../cli/commands/module-update';
27
+ import { getDb } from '../db/client';
28
+ import { modules } from '../db/schema';
29
+ import { RESPONDER_PROBE_EVENT } from './responder-probe';
30
+
31
+ const NO_SCHEMAS = defineEvents({});
32
+ const QUERY_TYPE = 'interview.required.module-upgrade:iptables.apply_breaking';
33
+
34
+ let dir: string;
35
+ let server: ReturnType<typeof Bun.serve>;
36
+ let registryUrl: string;
37
+
38
+ /** A responder that proves liveness but never answers the question. */
39
+ function probeOnlyResponder(busDbPath: string): { bus: Bus; close: () => void } {
40
+ const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
41
+ const watch = bus.watch(RESPONDER_PROBE_EVENT, async (event) => {
42
+ if (event.replyFor !== null) return;
43
+ bus.emitRaw(
44
+ `${event.type}.reply`,
45
+ { kind: 'daemon', emittedBy: 'park-test' },
46
+ { replyFor: event.id, emittedBy: 'park-test' },
47
+ );
48
+ });
49
+ return {
50
+ bus,
51
+ close: () => {
52
+ watch.close();
53
+ bus.close();
54
+ },
55
+ };
56
+ }
57
+
58
+ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
59
+
60
+ beforeEach(() => {
61
+ dir = mkdtempSync(join(tmpdir(), 'celilo-park-'));
62
+ process.env.CELILO_DB_PATH = join(dir, 'test.db');
63
+ process.env.CELILO_ORIGINAL_CWD = dir;
64
+ process.env.EVENT_BUS_DB = join(dir, 'events.db');
65
+
66
+ getDb()
67
+ .insert(modules)
68
+ .values({
69
+ id: 'iptables',
70
+ name: 'iptables',
71
+ sourcePath: join(dir, 'installed'),
72
+ version: '1.0.2+9',
73
+ manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
74
+ })
75
+ .run();
76
+
77
+ server = Bun.serve({
78
+ port: 0,
79
+ fetch(req) {
80
+ if (new URL(req.url).pathname === '/index/ip/ta/iptables') {
81
+ return new Response(
82
+ `${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
83
+ );
84
+ }
85
+ return new Response('not found', { status: 404 });
86
+ },
87
+ });
88
+ registryUrl = `http://localhost:${server.port}`;
89
+ });
90
+
91
+ afterEach(() => {
92
+ server.stop(true);
93
+ rmSync(dir, { recursive: true, force: true });
94
+ process.env.CELILO_DB_PATH = undefined;
95
+ process.env.CELILO_ORIGINAL_CWD = undefined;
96
+ process.env.EVENT_BUS_DB = undefined;
97
+ });
98
+
99
+ test('a command parks on an unanswered interview, then resumes with an answer given later', async () => {
100
+ const responder = probeOnlyResponder(join(dir, 'events.db'));
101
+ const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
102
+
103
+ try {
104
+ // Start the sweep but do NOT await it — it should block on the confirm.
105
+ let settled = false;
106
+ const sweep = handleModuleUpdate([], { registry: registryUrl }).then((r) => {
107
+ settled = true;
108
+ return r;
109
+ });
110
+
111
+ // 1. It parks. The sweep resolves in milliseconds if it does NOT park, so
112
+ // any interval well past the 250ms bus poll proves the point — no reason
113
+ // to hold a CI runner for seconds to say it.
114
+ await sleep(600);
115
+ expect(settled).toBe(false);
116
+
117
+ // 2. The question is on the bus, unanswered, and identifiable by event id.
118
+ const queries = observer.recentEvents({ type: QUERY_TYPE });
119
+ expect(queries.length).toBe(1);
120
+ const query = queries[0] as BusEvent;
121
+ const replies = observer.recentEvents({ type: `${QUERY_TYPE}.reply` });
122
+ expect(replies.length).toBe(0);
123
+
124
+ // 3. Answer it out-of-band, exactly as `celilo events reply <id> false` does.
125
+ observer.emitRaw(
126
+ `${QUERY_TYPE}.reply`,
127
+ { value: false },
128
+ { replyFor: query.id, emittedBy: 'claude-config-responder' },
129
+ );
130
+
131
+ // 4. It resumes AND uses the answer: `false` is a genuine decline, so the
132
+ // summary must say declined — not "NOT declined", which is what we'd see
133
+ // if it had failed rather than parked.
134
+ const result = await sweep;
135
+ const report = result.success ? (result.message ?? '') : (result.error ?? '');
136
+ expect(report).toContain('operator declined');
137
+ expect(report).not.toContain('NOT declined');
138
+ expect(result.success).toBe(true);
139
+ } finally {
140
+ observer.close();
141
+ responder.close();
142
+ }
143
+ }, 30_000);
144
+
145
+ /**
146
+ * The instrument the original report reached for cannot see this question.
147
+ *
148
+ * `celilo events list-pending` is `bus.pendingDeliveries()` — it reads the
149
+ * `deliveries` table (subscriber fan-out), while an unanswered interview is a
150
+ * row in `events` awaiting a correlated reply. So "list-pending returned []"
151
+ * was never evidence about the interview either way. This pins that down so
152
+ * #609 builds its observability gate on something that can actually observe.
153
+ */
154
+ test('events list-pending cannot see a parked interview — it reads a different table', async () => {
155
+ const responder = probeOnlyResponder(join(dir, 'events.db'));
156
+ const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
157
+
158
+ try {
159
+ const sweep = handleModuleUpdate([], { registry: registryUrl });
160
+ await sleep(600);
161
+
162
+ // The question is genuinely there...
163
+ const queries = observer.recentEvents({ type: QUERY_TYPE });
164
+ expect(queries.length).toBe(1);
165
+
166
+ // ...and list-pending shows nothing, because it is looking elsewhere.
167
+ expect(observer.pendingDeliveries({ limit: 100 })).toHaveLength(0);
168
+
169
+ observer.emitRaw(
170
+ `${QUERY_TYPE}.reply`,
171
+ { value: false },
172
+ { replyFor: queries[0].id, emittedBy: 'park-test' },
173
+ );
174
+ await sweep;
175
+ } finally {
176
+ observer.close();
177
+ responder.close();
178
+ }
179
+ }, 30_000);