@celilo/cli 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +4 -2
- package/package.json +4 -4
- package/src/api/remote-client.test.ts +86 -2
- package/src/api/serve.ts +242 -38
- package/src/api/sessions.test.ts +196 -0
- package/src/api/sessions.ts +278 -0
- package/src/cli/commands/apt-upgrade.test.ts +20 -1
- package/src/cli/commands/apt-upgrade.ts +12 -2
- package/src/cli/commands/backup-sweep.ts +25 -9
- package/src/cli/commands/events.ts +150 -4
- package/src/cli/commands/module-update.test.ts +72 -1
- package/src/cli/commands/module-update.ts +68 -22
- package/src/cli/commands/system-migrate.test.ts +56 -0
- package/src/cli/commands/system-migrate.ts +52 -4
- package/src/cli/completion.ts +2 -0
- package/src/cli/index.ts +27 -3
- package/src/db/migration-status.test.ts +114 -0
- package/src/db/migration-status.ts +78 -0
- package/src/db/schema-introspection.ts +8 -1
- package/src/services/backup-metadata.ts +17 -0
- package/src/services/backup-staging.test.ts +98 -0
- package/src/services/backup-staging.ts +73 -1
- package/src/services/backup-sweep.test.ts +15 -0
- package/src/services/backup-sweep.ts +17 -1
- package/src/services/bus-interview-park.test.ts +179 -0
- package/src/services/bus-interview.ts +17 -6
- package/src/services/events-daemon.test.ts +244 -0
- package/src/services/events-daemon.ts +295 -8
- package/src/services/fleet-checks.test.ts +75 -4
- package/src/services/fleet-checks.ts +82 -12
- package/src/services/interview-errors.ts +37 -0
- package/src/services/remote-responder.test.ts +83 -0
- package/src/services/remote-responder.ts +31 -10
- package/src/services/responder-probe.ts +3 -1
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* celilo#604: a table COUNT reported "up to date" for a DB missing a migrated
|
|
3
|
+
* COLUMN, so the rollout's own stop-condition was uncheckable by the tools the
|
|
4
|
+
* runbook named. These assert the column case specifically.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { Database } from 'bun:sqlite';
|
|
8
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
9
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
10
|
+
import { tmpdir } from 'node:os';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { getMigrationStatus, readMigrationJournal } from './migration-status';
|
|
13
|
+
|
|
14
|
+
/** A journal + a `__drizzle_migrations` table, wired the way drizzle wires them. */
|
|
15
|
+
function seed(dir: string, entries: Array<{ when: number; tag: string }>, appliedWhens: number[]) {
|
|
16
|
+
mkdirSync(join(dir, 'meta'), { recursive: true });
|
|
17
|
+
writeFileSync(
|
|
18
|
+
join(dir, 'meta', '_journal.json'),
|
|
19
|
+
JSON.stringify({ version: '7', dialect: 'sqlite', entries }),
|
|
20
|
+
);
|
|
21
|
+
const db = new Database(':memory:');
|
|
22
|
+
db.run(
|
|
23
|
+
'CREATE TABLE `__drizzle_migrations` (id INTEGER PRIMARY KEY, hash TEXT, created_at NUMERIC)',
|
|
24
|
+
);
|
|
25
|
+
for (const when of appliedWhens) {
|
|
26
|
+
db.run('INSERT INTO `__drizzle_migrations` (hash, created_at) VALUES (?, ?)', [
|
|
27
|
+
`h${when}`,
|
|
28
|
+
when,
|
|
29
|
+
]);
|
|
30
|
+
}
|
|
31
|
+
return db;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
describe('getMigrationStatus', () => {
|
|
35
|
+
let dir: string;
|
|
36
|
+
beforeEach(() => {
|
|
37
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-migstatus-'));
|
|
38
|
+
});
|
|
39
|
+
afterEach(() => rmSync(dir, { recursive: true, force: true }));
|
|
40
|
+
|
|
41
|
+
it('names the latest applied migration and every pending one', () => {
|
|
42
|
+
const db = seed(
|
|
43
|
+
dir,
|
|
44
|
+
[
|
|
45
|
+
{ when: 100, tag: '0018_drop_alert_policy_snapshot' },
|
|
46
|
+
{ when: 200, tag: '0019_backup_pid' },
|
|
47
|
+
{ when: 300, tag: '0020_future' },
|
|
48
|
+
],
|
|
49
|
+
[100, 200],
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
const status = getMigrationStatus(db, dir);
|
|
53
|
+
|
|
54
|
+
expect(status.appliedCount).toBe(2);
|
|
55
|
+
expect(status.latestApplied).toBe('0019_backup_pid');
|
|
56
|
+
expect(status.pending).toEqual(['0020_future']);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('reports nothing pending once every journal entry is applied', () => {
|
|
60
|
+
const db = seed(dir, [{ when: 100, tag: '0000_init' }], [100]);
|
|
61
|
+
const status = getMigrationStatus(db, dir);
|
|
62
|
+
expect(status.pending).toEqual([]);
|
|
63
|
+
expect(status.latestApplied).toBe('0000_init');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('reports a DB that has never been migrated as all-pending', () => {
|
|
67
|
+
mkdirSync(join(dir, 'meta'), { recursive: true });
|
|
68
|
+
writeFileSync(
|
|
69
|
+
join(dir, 'meta', '_journal.json'),
|
|
70
|
+
JSON.stringify({ entries: [{ when: 1, tag: '0000_init' }] }),
|
|
71
|
+
);
|
|
72
|
+
const status = getMigrationStatus(new Database(':memory:'), dir);
|
|
73
|
+
expect(status.appliedCount).toBe(0);
|
|
74
|
+
expect(status.latestApplied).toBeNull();
|
|
75
|
+
expect(status.pending).toEqual(['0000_init']);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('counts columns, not just tables — the signal a column migration needs', () => {
|
|
79
|
+
const db = seed(dir, [], []);
|
|
80
|
+
const status = getMigrationStatus(db, dir);
|
|
81
|
+
// Every schema table is missing here, so the counts are the code's totals.
|
|
82
|
+
expect(status.tableCount).toBeGreaterThan(0);
|
|
83
|
+
expect(status.columnCount).toBeGreaterThan(status.tableCount);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('reads journal entries oldest-first regardless of file order', () => {
|
|
87
|
+
mkdirSync(join(dir, 'meta'), { recursive: true });
|
|
88
|
+
writeFileSync(
|
|
89
|
+
join(dir, 'meta', '_journal.json'),
|
|
90
|
+
JSON.stringify({
|
|
91
|
+
entries: [
|
|
92
|
+
{ when: 300, tag: 'c' },
|
|
93
|
+
{ when: 100, tag: 'a' },
|
|
94
|
+
],
|
|
95
|
+
}),
|
|
96
|
+
);
|
|
97
|
+
expect(readMigrationJournal(dir).map((e) => e.tag)).toEqual(['a', 'c']);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('returns an empty journal when the folder has none', () => {
|
|
101
|
+
expect(readMigrationJournal(join(dir, 'nope'))).toEqual([]);
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
describe('getMigrationStatus against the real migrations folder', () => {
|
|
106
|
+
it('sees backups.pid as missing when the column migration did not land', () => {
|
|
107
|
+
// The exact celilo#604 shape: every table present, one migrated column not.
|
|
108
|
+
const db = new Database(':memory:');
|
|
109
|
+
db.run('CREATE TABLE backups (id TEXT PRIMARY KEY)');
|
|
110
|
+
const status = getMigrationStatus(db, join(import.meta.dir, '../../drizzle'));
|
|
111
|
+
expect(status.missingColumns).toContain('backups.pid');
|
|
112
|
+
expect(status.pending).toContain('0019_backup_pid');
|
|
113
|
+
});
|
|
114
|
+
});
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only migration interrogation (celilo#604).
|
|
3
|
+
*
|
|
4
|
+
* `system migrate` used to report "Schema current: 35 tables" — a table COUNT,
|
|
5
|
+
* which cannot distinguish "the column migration applied" from "nothing
|
|
6
|
+
* happened". 0019_backup_pid adds a column; a rollout runbook asserting
|
|
7
|
+
* "applied 19 → 20, backups.pid present" had no product surface to check it
|
|
8
|
+
* against and had to reach for `sqlite3` over SSH.
|
|
9
|
+
*
|
|
10
|
+
* This names migrations. Drizzle's `__drizzle_migrations.created_at` is the
|
|
11
|
+
* journal entry's `when`, so the join back to a human tag is exact.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { Database } from 'bun:sqlite';
|
|
15
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
16
|
+
import { join } from 'node:path';
|
|
17
|
+
import { findSchemaDrift } from './schema-introspection';
|
|
18
|
+
|
|
19
|
+
export interface MigrationStatus {
|
|
20
|
+
/** Rows in `__drizzle_migrations` — what the runbook calls "applied count". */
|
|
21
|
+
appliedCount: number;
|
|
22
|
+
/** Tag of the newest applied migration, e.g. `0019_backup_pid`. */
|
|
23
|
+
latestApplied: string | null;
|
|
24
|
+
/** Tags present on disk that this DB has not applied, oldest first. */
|
|
25
|
+
pending: string[];
|
|
26
|
+
/** Tables the running code declares that the DB lacks. */
|
|
27
|
+
missingTables: string[];
|
|
28
|
+
/** `table.column` the running code declares that the DB lacks. */
|
|
29
|
+
missingColumns: string[];
|
|
30
|
+
tableCount: number;
|
|
31
|
+
columnCount: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface JournalEntry {
|
|
35
|
+
when: number;
|
|
36
|
+
tag: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Journal entries (oldest first), or [] when the folder has no journal. */
|
|
40
|
+
export function readMigrationJournal(migrationsFolder: string): JournalEntry[] {
|
|
41
|
+
const path = join(migrationsFolder, 'meta', '_journal.json');
|
|
42
|
+
if (!existsSync(path)) return [];
|
|
43
|
+
const parsed = JSON.parse(readFileSync(path, 'utf-8')) as { entries?: JournalEntry[] };
|
|
44
|
+
return [...(parsed.entries ?? [])].sort((a, b) => a.when - b.when);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** `created_at` timestamps recorded in `__drizzle_migrations` (empty if untracked). */
|
|
48
|
+
function appliedTimestamps(sqlite: Database): number[] {
|
|
49
|
+
try {
|
|
50
|
+
return sqlite
|
|
51
|
+
.query<{ created_at: number }, []>(
|
|
52
|
+
'SELECT created_at FROM `__drizzle_migrations` ORDER BY created_at',
|
|
53
|
+
)
|
|
54
|
+
.all()
|
|
55
|
+
.map((r) => r.created_at);
|
|
56
|
+
} catch {
|
|
57
|
+
// No migrations table — a DB that has never been through the migrator.
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function getMigrationStatus(sqlite: Database, migrationsFolder: string): MigrationStatus {
|
|
63
|
+
const journal = readMigrationJournal(migrationsFolder);
|
|
64
|
+
const applied = new Set(appliedTimestamps(sqlite));
|
|
65
|
+
const appliedTags = journal.filter((e) => applied.has(e.when)).map((e) => e.tag);
|
|
66
|
+
const pending = journal.filter((e) => !applied.has(e.when)).map((e) => e.tag);
|
|
67
|
+
const drift = findSchemaDrift(sqlite);
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
appliedCount: applied.size,
|
|
71
|
+
latestApplied: appliedTags.at(-1) ?? null,
|
|
72
|
+
pending,
|
|
73
|
+
missingTables: drift.missingTables,
|
|
74
|
+
missingColumns: drift.missingColumns,
|
|
75
|
+
tableCount: drift.tableCount,
|
|
76
|
+
columnCount: drift.columnCount,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -20,6 +20,12 @@ export interface SchemaDrift {
|
|
|
20
20
|
missingColumns: string[];
|
|
21
21
|
/** Total number of tables the code's schema declares. */
|
|
22
22
|
tableCount: number;
|
|
23
|
+
/**
|
|
24
|
+
* Total number of columns the code's schema declares. Reported alongside
|
|
25
|
+
* tableCount so the doctor can say what it actually checked — a table count
|
|
26
|
+
* alone reads as "columns unverified" even when they were (celilo#604).
|
|
27
|
+
*/
|
|
28
|
+
columnCount: number;
|
|
23
29
|
}
|
|
24
30
|
|
|
25
31
|
/** Every table name + column names the drizzle schema declares. */
|
|
@@ -84,5 +90,6 @@ export function findSchemaDrift(sqlite: Database): SchemaDrift {
|
|
|
84
90
|
if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
|
|
85
91
|
}
|
|
86
92
|
}
|
|
87
|
-
|
|
93
|
+
const columnCount = tables.reduce((n, t) => n + t.columns.length, 0);
|
|
94
|
+
return { missingTables, missingColumns, tableCount: tables.length, columnCount };
|
|
88
95
|
}
|
|
@@ -156,6 +156,23 @@ export function listBackups(options?: {
|
|
|
156
156
|
return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
/**
|
|
160
|
+
* Every backup record still claiming to be in progress, oldest first.
|
|
161
|
+
*
|
|
162
|
+
* Feeds the record-resolution pass (`resolveAbandonedBackups`), which corrects
|
|
163
|
+
* the ones whose process is gone. Unbounded on purpose — there is no sensible
|
|
164
|
+
* limit on "how many lies to fix", and celilo-mgr had 107 of them.
|
|
165
|
+
*/
|
|
166
|
+
export function listInProgressBackups(): Backup[] {
|
|
167
|
+
const db = getDb();
|
|
168
|
+
return db
|
|
169
|
+
.select()
|
|
170
|
+
.from(backups)
|
|
171
|
+
.where(eq(backups.status, 'in_progress'))
|
|
172
|
+
.orderBy(backups.startedAt)
|
|
173
|
+
.all();
|
|
174
|
+
}
|
|
175
|
+
|
|
159
176
|
/**
|
|
160
177
|
* List completed backups for a specific module, ordered newest first
|
|
161
178
|
*/
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import {
|
|
3
|
+
ABANDONED_BACKUP_MESSAGE,
|
|
4
|
+
type InProgressBackup,
|
|
3
5
|
type ReapStagingDeps,
|
|
6
|
+
type ResolveAbandonedDeps,
|
|
4
7
|
STAGING_PREFIX,
|
|
5
8
|
STAGING_TTL_MS,
|
|
6
9
|
type StagingOwner,
|
|
7
10
|
reapOrphanedStaging,
|
|
11
|
+
resolveAbandonedBackups,
|
|
8
12
|
stagingDirFor,
|
|
9
13
|
} from './backup-staging';
|
|
10
14
|
|
|
@@ -132,3 +136,97 @@ describe('reapOrphanedStaging', () => {
|
|
|
132
136
|
expect(report.kept).toEqual([locked]);
|
|
133
137
|
});
|
|
134
138
|
});
|
|
139
|
+
|
|
140
|
+
function recordDeps(
|
|
141
|
+
records: InProgressBackup[],
|
|
142
|
+
runnablePids: number[] = [],
|
|
143
|
+
): ResolveAbandonedDeps & { failed: Array<{ id: string; message: string }> } {
|
|
144
|
+
const failed: Array<{ id: string; message: string }> = [];
|
|
145
|
+
return {
|
|
146
|
+
failed,
|
|
147
|
+
listInProgress: () => records,
|
|
148
|
+
isPidRunnable: (pid) => runnablePids.includes(pid),
|
|
149
|
+
fail: (id, message) => failed.push({ id, message }),
|
|
150
|
+
now: () => NOW,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function record(over: Partial<InProgressBackup> = {}): InProgressBackup {
|
|
155
|
+
return { id: 'rec-1', pid: 222, startedAt: new Date(NOW - 60_000), ...over };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
describe('resolveAbandonedBackups', () => {
|
|
159
|
+
// THE REGRESSION THIS EXISTS FOR (#616). Every earlier test supplied a
|
|
160
|
+
// staging directory, so the coupled implementation passed all of them while
|
|
161
|
+
// leaving 107 real records stranded. Nothing here mentions staging at all —
|
|
162
|
+
// that is the point.
|
|
163
|
+
test('resolves a dead record even though no staging directory exists', () => {
|
|
164
|
+
const d = recordDeps([record({ id: 'orphan', pid: 999 })], []);
|
|
165
|
+
const report = resolveAbandonedBackups(d);
|
|
166
|
+
|
|
167
|
+
expect(report.resolved).toEqual(['orphan']);
|
|
168
|
+
expect(d.failed).toEqual([{ id: 'orphan', message: ABANDONED_BACKUP_MESSAGE }]);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// All 107 rows on celilo-mgr predate the pid column. Age is the only signal
|
|
172
|
+
// they carry, so they must resolve on it rather than linger forever.
|
|
173
|
+
test('resolves a pid-less record once it is past the TTL', () => {
|
|
174
|
+
const d = recordDeps([
|
|
175
|
+
record({ id: 'pre-migration', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
expect(resolveAbandonedBackups(d).resolved).toEqual(['pre-migration']);
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test('keeps a pid-less record that is still inside the TTL', () => {
|
|
182
|
+
const d = recordDeps([record({ id: 'recent', pid: null })]);
|
|
183
|
+
const report = resolveAbandonedBackups(d);
|
|
184
|
+
|
|
185
|
+
expect(report.resolved).toEqual([]);
|
|
186
|
+
expect(report.kept).toEqual(['recent']);
|
|
187
|
+
expect(d.failed).toEqual([]);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// Must never regress: a backup mid-flight is not abandoned.
|
|
191
|
+
test('KEEPS a record whose process is alive', () => {
|
|
192
|
+
const d = recordDeps([record({ id: 'live', pid: 222 })], [222]);
|
|
193
|
+
const report = resolveAbandonedBackups(d);
|
|
194
|
+
|
|
195
|
+
expect(report.resolved).toEqual([]);
|
|
196
|
+
expect(report.kept).toEqual(['live']);
|
|
197
|
+
expect(d.failed).toEqual([]);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test('resolves a record past the TTL even when its pid looks alive', () => {
|
|
201
|
+
// Same pid-reuse guard the reaper applies.
|
|
202
|
+
const d = recordDeps(
|
|
203
|
+
[record({ id: 'stale', pid: 222, startedAt: new Date(NOW - STAGING_TTL_MS - 1) })],
|
|
204
|
+
[222],
|
|
205
|
+
);
|
|
206
|
+
|
|
207
|
+
expect(resolveAbandonedBackups(d).resolved).toEqual(['stale']);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
test('sorts a mixed set without touching the live one', () => {
|
|
211
|
+
const d = recordDeps(
|
|
212
|
+
[
|
|
213
|
+
record({ id: 'dead', pid: 999 }),
|
|
214
|
+
record({ id: 'live', pid: 222 }),
|
|
215
|
+
record({ id: 'old', pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
|
|
216
|
+
],
|
|
217
|
+
[222],
|
|
218
|
+
);
|
|
219
|
+
const report = resolveAbandonedBackups(d);
|
|
220
|
+
|
|
221
|
+
expect(report.resolved.sort()).toEqual(['dead', 'old']);
|
|
222
|
+
expect(report.kept).toEqual(['live']);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
test('nothing in progress is a no-op', () => {
|
|
226
|
+
const d = recordDeps([]);
|
|
227
|
+
const report = resolveAbandonedBackups(d);
|
|
228
|
+
|
|
229
|
+
expect(report).toEqual({ resolved: [], kept: [] });
|
|
230
|
+
expect(d.failed).toEqual([]);
|
|
231
|
+
});
|
|
232
|
+
});
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Cleaning up after backups whose process died — the staging they left on disk,
|
|
3
|
+
* and the records that still claim they are running.
|
|
4
|
+
*
|
|
5
|
+
* The two are separate obligations that share a predicate, NOT one obligation
|
|
6
|
+
* with two effects. Coupling them is exactly the bug in #616; see
|
|
7
|
+
* `resolveAbandonedBackups`.
|
|
3
8
|
*
|
|
4
9
|
* `backup-create.ts` assembles every envelope in a temp directory and removes
|
|
5
10
|
* it in a `finally`. That is correct and it is not enough: a `finally` does not
|
|
@@ -114,6 +119,73 @@ export interface ReapStagingReport {
|
|
|
114
119
|
ignored: string[];
|
|
115
120
|
}
|
|
116
121
|
|
|
122
|
+
/** An `in_progress` backup record, as seen by the record-resolution pass. */
|
|
123
|
+
export interface InProgressBackup {
|
|
124
|
+
id: string;
|
|
125
|
+
/** Null for records written before backups recorded their pid. */
|
|
126
|
+
pid: number | null;
|
|
127
|
+
startedAt: Date;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface ResolveAbandonedDeps {
|
|
131
|
+
listInProgress(): InProgressBackup[];
|
|
132
|
+
isPidRunnable(pid: number): boolean;
|
|
133
|
+
fail(recordId: string, message: string): void;
|
|
134
|
+
now(): number;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface ResolveAbandonedReport {
|
|
138
|
+
/** Records corrected from `in_progress` to failed. */
|
|
139
|
+
resolved: string[];
|
|
140
|
+
/** Left alone — a live backup owns them. */
|
|
141
|
+
kept: string[];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Correct records that still claim to be running after their process died.
|
|
146
|
+
*
|
|
147
|
+
* Deliberately INDEPENDENT of staging. The first version of this resolved
|
|
148
|
+
* records only as a side effect of reclaiming their staging directory, which
|
|
149
|
+
* was efficient — one liveness lookup serving two places — and strictly
|
|
150
|
+
* narrower than the requirement. A record whose staging is already gone was
|
|
151
|
+
* never visited, so it stayed `in_progress` forever: 107 such rows on
|
|
152
|
+
* celilo-mgr, the oldest from June, none of them reachable by the reaper
|
|
153
|
+
* because their directories had been cleared by hand (#616).
|
|
154
|
+
*
|
|
155
|
+
* That is not an edge case. `/tmp` is declared `D` in tmpfiles.d — cleared on
|
|
156
|
+
* boot — so ANY backup killed before a reboot loses its staging and becomes
|
|
157
|
+
* permanently unresolvable under the coupled design. Reclaiming disk and
|
|
158
|
+
* correcting records are two obligations that happen to share a predicate, not
|
|
159
|
+
* one obligation with two effects.
|
|
160
|
+
*
|
|
161
|
+
* The predicate itself is shared rather than reimplemented: this calls the same
|
|
162
|
+
* `reclaimReason` the reaper uses, so the TTL-versus-pid-reuse reasoning has
|
|
163
|
+
* exactly one home. A record is only ever resolved when its owner is provably
|
|
164
|
+
* gone; a live backup is left alone.
|
|
165
|
+
*/
|
|
166
|
+
export function resolveAbandonedBackups(deps: ResolveAbandonedDeps): ResolveAbandonedReport {
|
|
167
|
+
const report: ResolveAbandonedReport = { resolved: [], kept: [] };
|
|
168
|
+
|
|
169
|
+
for (const record of deps.listInProgress()) {
|
|
170
|
+
const reason = reclaimReason(
|
|
171
|
+
{ status: 'in_progress', pid: record.pid, startedAt: record.startedAt },
|
|
172
|
+
deps,
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
// `record-absent` and `record-terminal` are unreachable here — every row
|
|
176
|
+
// came from a query for in-progress records — so any reason at all means
|
|
177
|
+
// the owner is gone.
|
|
178
|
+
if (reason) {
|
|
179
|
+
deps.fail(record.id, ABANDONED_BACKUP_MESSAGE);
|
|
180
|
+
report.resolved.push(record.id);
|
|
181
|
+
} else {
|
|
182
|
+
report.kept.push(record.id);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return report;
|
|
187
|
+
}
|
|
188
|
+
|
|
117
189
|
/**
|
|
118
190
|
* Decide, for one staging directory, whether its owner is gone.
|
|
119
191
|
*
|
|
@@ -36,6 +36,7 @@ function deps(
|
|
|
36
36
|
pruned.push(id);
|
|
37
37
|
},
|
|
38
38
|
reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
|
|
39
|
+
resolveAbandonedRecords: () => ({ resolved: [], kept: [] }),
|
|
39
40
|
...overrides,
|
|
40
41
|
};
|
|
41
42
|
}
|
|
@@ -67,6 +68,20 @@ describe('runBackupSweep', () => {
|
|
|
67
68
|
expect(report.staging.reclaimed).toHaveLength(1);
|
|
68
69
|
});
|
|
69
70
|
|
|
71
|
+
// The sweep must resolve records whether or not the reaper found anything —
|
|
72
|
+
// that independence IS the fix for #616.
|
|
73
|
+
test('corrects abandoned records even when no staging was reclaimed', async () => {
|
|
74
|
+
const report = await runBackupSweep(
|
|
75
|
+
deps([moduleWith('forgejo', 'daily')], {
|
|
76
|
+
reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
|
|
77
|
+
resolveAbandonedRecords: () => ({ resolved: ['rec-a', 'rec-b'], kept: [] }),
|
|
78
|
+
}),
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
expect(report.staging.reclaimed).toEqual([]);
|
|
82
|
+
expect(report.records.resolved).toEqual(['rec-a', 'rec-b']);
|
|
83
|
+
});
|
|
84
|
+
|
|
70
85
|
test('reclaims staging even when no module is due to back up', async () => {
|
|
71
86
|
const report = await runBackupSweep(
|
|
72
87
|
deps([moduleWith('forgejo', 'daily')], {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
|
|
19
19
|
import type { ModuleManifest } from '../manifest/schema';
|
|
20
20
|
import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
|
|
21
|
-
import type { ReapStagingReport } from './backup-staging';
|
|
21
|
+
import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
|
|
22
22
|
import { InFlightError } from './module-operations';
|
|
23
23
|
|
|
24
24
|
export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
|
|
@@ -110,11 +110,21 @@ export interface BackupSweepDeps {
|
|
|
110
110
|
prune(module: BackupSweepModule): Promise<void>;
|
|
111
111
|
/** Reclaim staging left by backups whose process died. See backup-staging.ts. */
|
|
112
112
|
reapStaging(): ReapStagingReport;
|
|
113
|
+
/**
|
|
114
|
+
* Correct records that still claim to be running after their process died.
|
|
115
|
+
*
|
|
116
|
+
* Separate from `reapStaging` on purpose. Deriving this from what the reaper
|
|
117
|
+
* happened to reclaim left records stranded forever once their staging was
|
|
118
|
+
* gone — #616.
|
|
119
|
+
*/
|
|
120
|
+
resolveAbandonedRecords(): ResolveAbandonedReport;
|
|
113
121
|
}
|
|
114
122
|
|
|
115
123
|
export interface BackupSweepReport {
|
|
116
124
|
/** Staging reclaimed before this pass created any of its own. */
|
|
117
125
|
staging: ReapStagingReport;
|
|
126
|
+
/** Records corrected from a stale `in_progress`. */
|
|
127
|
+
records: ResolveAbandonedReport;
|
|
118
128
|
backedUp: string[];
|
|
119
129
|
/** Explicit `schedule: manual` — the author opted out. */
|
|
120
130
|
skippedManual: string[];
|
|
@@ -131,8 +141,14 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
|
|
|
131
141
|
// another ~4 GB of it, not once we are finished with it.
|
|
132
142
|
const staging = deps.reapStaging();
|
|
133
143
|
|
|
144
|
+
// Independent of the reap above, and that independence is the fix for #616:
|
|
145
|
+
// a record whose staging is already gone is invisible to the reaper, so
|
|
146
|
+
// resolving records off the reaper's results left 107 of them stranded.
|
|
147
|
+
const records = deps.resolveAbandonedRecords();
|
|
148
|
+
|
|
134
149
|
const report: BackupSweepReport = {
|
|
135
150
|
staging,
|
|
151
|
+
records,
|
|
136
152
|
backedUp: [],
|
|
137
153
|
skippedManual: [],
|
|
138
154
|
skippedNotDue: [],
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does `busInterview` actually PARK?
|
|
3
|
+
*
|
|
4
|
+
* celilo#609's whole scope rests on one claim: a command blocked on an
|
|
5
|
+
* unanswered interview stays alive indefinitely and resumes when the question
|
|
6
|
+
* is answered later. If true, #609 needs no command-state serialization — only
|
|
7
|
+
* session lifetime and re-attach. If the command instead dies quietly, #609 is
|
|
8
|
+
* a much larger design.
|
|
9
|
+
*
|
|
10
|
+
* That claim was originally read off the code (`timeoutMs: 0`), which is not
|
|
11
|
+
* the same as watching it happen. This file watches it happen: a real
|
|
12
|
+
* `module update` sweep parks on a breaking-update confirm nobody answers,
|
|
13
|
+
* stays parked, and then resumes and uses the answer when one arrives by event
|
|
14
|
+
* id — the same reply `celilo events reply <id> <value>` emits.
|
|
15
|
+
*
|
|
16
|
+
* The responder here answers `responder.probe` but deliberately ignores the
|
|
17
|
+
* interview, which is exactly the #609 situation: a responder exists (the
|
|
18
|
+
* api-serve bridge), so the fail-fast guard passes, but nobody can decide.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { afterEach, beforeEach, expect, test } from 'bun:test';
|
|
22
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
23
|
+
import { tmpdir } from 'node:os';
|
|
24
|
+
import { join } from 'node:path';
|
|
25
|
+
import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bus';
|
|
26
|
+
import { handleModuleUpdate } from '../cli/commands/module-update';
|
|
27
|
+
import { getDb } from '../db/client';
|
|
28
|
+
import { modules } from '../db/schema';
|
|
29
|
+
import { RESPONDER_PROBE_EVENT } from './responder-probe';
|
|
30
|
+
|
|
31
|
+
const NO_SCHEMAS = defineEvents({});
|
|
32
|
+
const QUERY_TYPE = 'interview.required.module-upgrade:iptables.apply_breaking';
|
|
33
|
+
|
|
34
|
+
let dir: string;
|
|
35
|
+
let server: ReturnType<typeof Bun.serve>;
|
|
36
|
+
let registryUrl: string;
|
|
37
|
+
|
|
38
|
+
/** A responder that proves liveness but never answers the question. */
|
|
39
|
+
function probeOnlyResponder(busDbPath: string): { bus: Bus; close: () => void } {
|
|
40
|
+
const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
|
|
41
|
+
const watch = bus.watch(RESPONDER_PROBE_EVENT, async (event) => {
|
|
42
|
+
if (event.replyFor !== null) return;
|
|
43
|
+
bus.emitRaw(
|
|
44
|
+
`${event.type}.reply`,
|
|
45
|
+
{ kind: 'daemon', emittedBy: 'park-test' },
|
|
46
|
+
{ replyFor: event.id, emittedBy: 'park-test' },
|
|
47
|
+
);
|
|
48
|
+
});
|
|
49
|
+
return {
|
|
50
|
+
bus,
|
|
51
|
+
close: () => {
|
|
52
|
+
watch.close();
|
|
53
|
+
bus.close();
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
59
|
+
|
|
60
|
+
beforeEach(() => {
|
|
61
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-park-'));
|
|
62
|
+
process.env.CELILO_DB_PATH = join(dir, 'test.db');
|
|
63
|
+
process.env.CELILO_ORIGINAL_CWD = dir;
|
|
64
|
+
process.env.EVENT_BUS_DB = join(dir, 'events.db');
|
|
65
|
+
|
|
66
|
+
getDb()
|
|
67
|
+
.insert(modules)
|
|
68
|
+
.values({
|
|
69
|
+
id: 'iptables',
|
|
70
|
+
name: 'iptables',
|
|
71
|
+
sourcePath: join(dir, 'installed'),
|
|
72
|
+
version: '1.0.2+9',
|
|
73
|
+
manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
|
|
74
|
+
})
|
|
75
|
+
.run();
|
|
76
|
+
|
|
77
|
+
server = Bun.serve({
|
|
78
|
+
port: 0,
|
|
79
|
+
fetch(req) {
|
|
80
|
+
if (new URL(req.url).pathname === '/index/ip/ta/iptables') {
|
|
81
|
+
return new Response(
|
|
82
|
+
`${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
return new Response('not found', { status: 404 });
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
registryUrl = `http://localhost:${server.port}`;
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
afterEach(() => {
|
|
92
|
+
server.stop(true);
|
|
93
|
+
rmSync(dir, { recursive: true, force: true });
|
|
94
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
95
|
+
process.env.CELILO_ORIGINAL_CWD = undefined;
|
|
96
|
+
process.env.EVENT_BUS_DB = undefined;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('a command parks on an unanswered interview, then resumes with an answer given later', async () => {
|
|
100
|
+
const responder = probeOnlyResponder(join(dir, 'events.db'));
|
|
101
|
+
const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
// Start the sweep but do NOT await it — it should block on the confirm.
|
|
105
|
+
let settled = false;
|
|
106
|
+
const sweep = handleModuleUpdate([], { registry: registryUrl }).then((r) => {
|
|
107
|
+
settled = true;
|
|
108
|
+
return r;
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
// 1. It parks. The sweep resolves in milliseconds if it does NOT park, so
|
|
112
|
+
// any interval well past the 250ms bus poll proves the point — no reason
|
|
113
|
+
// to hold a CI runner for seconds to say it.
|
|
114
|
+
await sleep(600);
|
|
115
|
+
expect(settled).toBe(false);
|
|
116
|
+
|
|
117
|
+
// 2. The question is on the bus, unanswered, and identifiable by event id.
|
|
118
|
+
const queries = observer.recentEvents({ type: QUERY_TYPE });
|
|
119
|
+
expect(queries.length).toBe(1);
|
|
120
|
+
const query = queries[0] as BusEvent;
|
|
121
|
+
const replies = observer.recentEvents({ type: `${QUERY_TYPE}.reply` });
|
|
122
|
+
expect(replies.length).toBe(0);
|
|
123
|
+
|
|
124
|
+
// 3. Answer it out-of-band, exactly as `celilo events reply <id> false` does.
|
|
125
|
+
observer.emitRaw(
|
|
126
|
+
`${QUERY_TYPE}.reply`,
|
|
127
|
+
{ value: false },
|
|
128
|
+
{ replyFor: query.id, emittedBy: 'claude-config-responder' },
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
// 4. It resumes AND uses the answer: `false` is a genuine decline, so the
|
|
132
|
+
// summary must say declined — not "NOT declined", which is what we'd see
|
|
133
|
+
// if it had failed rather than parked.
|
|
134
|
+
const result = await sweep;
|
|
135
|
+
const report = result.success ? (result.message ?? '') : (result.error ?? '');
|
|
136
|
+
expect(report).toContain('operator declined');
|
|
137
|
+
expect(report).not.toContain('NOT declined');
|
|
138
|
+
expect(result.success).toBe(true);
|
|
139
|
+
} finally {
|
|
140
|
+
observer.close();
|
|
141
|
+
responder.close();
|
|
142
|
+
}
|
|
143
|
+
}, 30_000);
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The instrument the original report reached for cannot see this question.
|
|
147
|
+
*
|
|
148
|
+
* `celilo events list-pending` is `bus.pendingDeliveries()` — it reads the
|
|
149
|
+
* `deliveries` table (subscriber fan-out), while an unanswered interview is a
|
|
150
|
+
* row in `events` awaiting a correlated reply. So "list-pending returned []"
|
|
151
|
+
* was never evidence about the interview either way. This pins that down so
|
|
152
|
+
* #609 builds its observability gate on something that can actually observe.
|
|
153
|
+
*/
|
|
154
|
+
test('events list-pending cannot see a parked interview — it reads a different table', async () => {
|
|
155
|
+
const responder = probeOnlyResponder(join(dir, 'events.db'));
|
|
156
|
+
const observer = openBus({ dbPath: join(dir, 'events.db'), events: NO_SCHEMAS });
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
const sweep = handleModuleUpdate([], { registry: registryUrl });
|
|
160
|
+
await sleep(600);
|
|
161
|
+
|
|
162
|
+
// The question is genuinely there...
|
|
163
|
+
const queries = observer.recentEvents({ type: QUERY_TYPE });
|
|
164
|
+
expect(queries.length).toBe(1);
|
|
165
|
+
|
|
166
|
+
// ...and list-pending shows nothing, because it is looking elsewhere.
|
|
167
|
+
expect(observer.pendingDeliveries({ limit: 100 })).toHaveLength(0);
|
|
168
|
+
|
|
169
|
+
observer.emitRaw(
|
|
170
|
+
`${QUERY_TYPE}.reply`,
|
|
171
|
+
{ value: false },
|
|
172
|
+
{ replyFor: queries[0].id, emittedBy: 'park-test' },
|
|
173
|
+
);
|
|
174
|
+
await sweep;
|
|
175
|
+
} finally {
|
|
176
|
+
observer.close();
|
|
177
|
+
responder.close();
|
|
178
|
+
}
|
|
179
|
+
}, 30_000);
|