@celilo/cli 0.14.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +3 -30
- package/src/cli/completion.ts +18 -1
- package/src/cli/index.ts +11 -0
- package/src/db/schema.ts +5 -3
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +17 -2
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +44 -8
- package/src/services/alerting/inbound-poller.ts +65 -28
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
3
|
+
import type { BackupSchedule } from './backup-schedule';
|
|
4
|
+
import {
|
|
5
|
+
BACKUP_SWEEP_PATTERN,
|
|
6
|
+
BACKUP_SWEEP_SUBSCRIBER,
|
|
7
|
+
type BackupSweepDeps,
|
|
8
|
+
type BackupSweepModule,
|
|
9
|
+
ensureBackupSweepSubscriber,
|
|
10
|
+
runBackupSweep,
|
|
11
|
+
} from './backup-sweep';
|
|
12
|
+
import { InFlightError } from './module-operations';
|
|
13
|
+
|
|
14
|
+
function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
|
|
15
|
+
const manifest = {
|
|
16
|
+
id,
|
|
17
|
+
hooks: { on_backup: { script: 'backup.ts' } },
|
|
18
|
+
...(schedule ? { backup: { schedule } } : {}),
|
|
19
|
+
} as unknown as ModuleManifest;
|
|
20
|
+
return { id, manifest };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function deps(
|
|
24
|
+
modules: BackupSweepModule[],
|
|
25
|
+
overrides: Partial<BackupSweepDeps> = {},
|
|
26
|
+
): BackupSweepDeps & { pruned: string[] } {
|
|
27
|
+
const pruned: string[] = [];
|
|
28
|
+
return {
|
|
29
|
+
pruned,
|
|
30
|
+
listEligible: () => modules,
|
|
31
|
+
isDue: () => true,
|
|
32
|
+
backup: async () => ({ success: true }),
|
|
33
|
+
prune: async ({ id }) => {
|
|
34
|
+
pruned.push(id);
|
|
35
|
+
},
|
|
36
|
+
...overrides,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe('runBackupSweep', () => {
|
|
41
|
+
test('backs up a module whose declared cadence is due', async () => {
|
|
42
|
+
const d = deps([moduleWith('authentik', 'daily')]);
|
|
43
|
+
const report = await runBackupSweep(d);
|
|
44
|
+
|
|
45
|
+
expect(report.backedUp).toEqual(['authentik']);
|
|
46
|
+
expect(report.failures).toEqual([]);
|
|
47
|
+
expect(d.pruned).toEqual(['authentik']);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('skips a module that is not yet due', async () => {
|
|
51
|
+
const report = await runBackupSweep(
|
|
52
|
+
deps([moduleWith('authentik', 'daily')], { isDue: () => false }),
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
expect(report.backedUp).toEqual([]);
|
|
56
|
+
expect(report.skippedNotDue).toEqual(['authentik']);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test('never auto-backs-up an explicit manual schedule', async () => {
|
|
60
|
+
const report = await runBackupSweep(deps([moduleWith('scratch', 'manual')]));
|
|
61
|
+
|
|
62
|
+
expect(report.backedUp).toEqual([]);
|
|
63
|
+
expect(report.skippedManual).toEqual(['scratch']);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('an undeclared schedule is backed up, not treated as manual', async () => {
|
|
67
|
+
// The regression this whole subsystem exists for: forgejo and signal
|
|
68
|
+
// declare no `backup:` block and had never been backed up.
|
|
69
|
+
const report = await runBackupSweep(deps([moduleWith('forgejo')]));
|
|
70
|
+
|
|
71
|
+
expect(report.backedUp).toEqual(['forgejo']);
|
|
72
|
+
expect(report.skippedManual).toEqual([]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test('a held operation lock is a skip, not a failure, and stops the pass', async () => {
|
|
76
|
+
const d = deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
|
|
77
|
+
backup: async () => {
|
|
78
|
+
throw new InFlightError([]);
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
const report = await runBackupSweep(d);
|
|
82
|
+
|
|
83
|
+
expect(report.skippedLocked).toEqual(['authentik']);
|
|
84
|
+
expect(report.failures).toEqual([]);
|
|
85
|
+
expect(report.backedUp).toEqual([]);
|
|
86
|
+
expect(d.pruned).toEqual([]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('a failed backup is recorded and the remaining modules still run', async () => {
|
|
90
|
+
const report = await runBackupSweep(
|
|
91
|
+
deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
|
|
92
|
+
backup: async (moduleId) =>
|
|
93
|
+
moduleId === 'authentik' ? { success: false, error: 'hook exited 1' } : { success: true },
|
|
94
|
+
}),
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'hook exited 1' }]);
|
|
98
|
+
expect(report.backedUp).toEqual(['forgejo']);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('a thrown storage error fails only that module', async () => {
|
|
102
|
+
const report = await runBackupSweep(
|
|
103
|
+
deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
|
|
104
|
+
backup: async (moduleId) => {
|
|
105
|
+
if (moduleId === 'authentik') throw new Error('Storage not verified');
|
|
106
|
+
return { success: true };
|
|
107
|
+
},
|
|
108
|
+
}),
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'Storage not verified' }]);
|
|
112
|
+
expect(report.backedUp).toEqual(['forgejo']);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('does not prune when a backup failed', async () => {
|
|
116
|
+
const d = deps([moduleWith('authentik', 'daily')], {
|
|
117
|
+
backup: async () => ({ success: false, error: 'nope' }),
|
|
118
|
+
});
|
|
119
|
+
await runBackupSweep(d);
|
|
120
|
+
|
|
121
|
+
expect(d.pruned).toEqual([]);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
describe('ensureBackupSweepSubscriber', () => {
|
|
126
|
+
test('registers the hourly sweep handler', () => {
|
|
127
|
+
const calls: Array<{
|
|
128
|
+
name: string;
|
|
129
|
+
pattern: string;
|
|
130
|
+
handler: string;
|
|
131
|
+
registeredBy?: string;
|
|
132
|
+
}> = [];
|
|
133
|
+
ensureBackupSweepSubscriber({
|
|
134
|
+
subscribe: (options) => calls.push(options),
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
expect(calls).toEqual([
|
|
138
|
+
{
|
|
139
|
+
name: BACKUP_SWEEP_SUBSCRIBER,
|
|
140
|
+
pattern: BACKUP_SWEEP_PATTERN,
|
|
141
|
+
handler: 'celilo backup sweep',
|
|
142
|
+
registeredBy: 'celilo-backup',
|
|
143
|
+
},
|
|
144
|
+
]);
|
|
145
|
+
// 1h is the coarsest tick that can still serve an `hourly` cadence.
|
|
146
|
+
expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
|
|
147
|
+
});
|
|
148
|
+
});
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The thing that actually runs backups on a schedule.
|
|
3
|
+
*
|
|
4
|
+
* A manifest's `backup.schedule` was decorative until this existed: the drift
|
|
5
|
+
* audit reported staleness and `celilo module backup` needed a human. This is
|
|
6
|
+
* the pass that makes a module declaring `daily` get backed up daily with
|
|
7
|
+
* nobody watching.
|
|
8
|
+
*
|
|
9
|
+
* No new scheduling mechanism. Registered as an ordinary bus subscriber whose
|
|
10
|
+
* handler is `celilo backup sweep` against `timer.tick.1h`, exactly like the
|
|
11
|
+
* alerting sweep (`services/alerting/monitors.ts`) — the timer already exists,
|
|
12
|
+
* already survives restarts, and already has retry and dedup. One hour is the
|
|
13
|
+
* coarsest tick that can still serve an `hourly` cadence; `1d` cannot.
|
|
14
|
+
*
|
|
15
|
+
* Dependencies are injected so the pass itself tests with no database, no
|
|
16
|
+
* storage, and no hook execution.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
20
|
+
import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
|
|
21
|
+
import { InFlightError } from './module-operations';
|
|
22
|
+
|
|
23
|
+
export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
|
|
24
|
+
export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
|
|
25
|
+
|
|
26
|
+
export interface SubscriberRegistrar {
|
|
27
|
+
subscribe(options: {
|
|
28
|
+
name: string;
|
|
29
|
+
pattern: string;
|
|
30
|
+
handler: string;
|
|
31
|
+
registeredBy?: string;
|
|
32
|
+
}): unknown;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
|
|
37
|
+
* every module install and update.
|
|
38
|
+
*/
|
|
39
|
+
export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
|
|
40
|
+
bus.subscribe({
|
|
41
|
+
name: BACKUP_SWEEP_SUBSCRIBER,
|
|
42
|
+
pattern: BACKUP_SWEEP_PATTERN,
|
|
43
|
+
handler: 'celilo backup sweep',
|
|
44
|
+
registeredBy: 'celilo-backup',
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface BackupSweepModule {
|
|
49
|
+
id: string;
|
|
50
|
+
manifest: ModuleManifest;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface BackupSweepDeps {
|
|
54
|
+
/** Installed modules that declare an `on_backup` hook. */
|
|
55
|
+
listEligible(): BackupSweepModule[];
|
|
56
|
+
isDue(moduleId: string, schedule: BackupSchedule): boolean;
|
|
57
|
+
backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
|
|
58
|
+
/** Apply the module's declared retention. No-op when it declares none. */
|
|
59
|
+
prune(module: BackupSweepModule): Promise<void>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface BackupSweepReport {
|
|
63
|
+
backedUp: string[];
|
|
64
|
+
/** Explicit `schedule: manual` — the author opted out. */
|
|
65
|
+
skippedManual: string[];
|
|
66
|
+
skippedNotDue: string[];
|
|
67
|
+
/** Another module operation held the lock. Not a failure; retried next tick. */
|
|
68
|
+
skippedLocked: string[];
|
|
69
|
+
failures: Array<{ moduleId: string; error: string }>;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
|
|
73
|
+
const report: BackupSweepReport = {
|
|
74
|
+
backedUp: [],
|
|
75
|
+
skippedManual: [],
|
|
76
|
+
skippedNotDue: [],
|
|
77
|
+
skippedLocked: [],
|
|
78
|
+
failures: [],
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
for (const module of deps.listEligible()) {
|
|
82
|
+
const schedule = effectiveBackupSchedule(module.manifest);
|
|
83
|
+
if (schedule === 'manual') {
|
|
84
|
+
report.skippedManual.push(module.id);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
if (!deps.isDue(module.id, schedule)) {
|
|
88
|
+
report.skippedNotDue.push(module.id);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
let result: { success: boolean; error?: string };
|
|
93
|
+
try {
|
|
94
|
+
result = await deps.backup(module.id);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
// Refusing to run while a deploy/restore is in flight is correct, and a
|
|
97
|
+
// scheduled run must not bypass it. The lock is global, so once it is
|
|
98
|
+
// held every remaining module would refuse identically — stop the pass
|
|
99
|
+
// rather than collect the same refusal N times. The next tick retries.
|
|
100
|
+
if (error instanceof InFlightError) {
|
|
101
|
+
report.skippedLocked.push(module.id);
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
// Everything else — an unverified storage destination, a missing default
|
|
105
|
+
// — throws before a backup row exists. Record it against the module and
|
|
106
|
+
// keep going; one module's failure must not cancel the rest.
|
|
107
|
+
report.failures.push({
|
|
108
|
+
moduleId: module.id,
|
|
109
|
+
error: error instanceof Error ? error.message : String(error),
|
|
110
|
+
});
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (!result.success) {
|
|
115
|
+
report.failures.push({ moduleId: module.id, error: result.error ?? 'backup failed' });
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
report.backedUp.push(module.id);
|
|
120
|
+
await deps.prune(module);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return report;
|
|
124
|
+
}
|
|
@@ -3,14 +3,27 @@
|
|
|
3
3
|
* openspec/changes/build-bus-poll-cd/proposal.md).
|
|
4
4
|
*
|
|
5
5
|
* CD is CI-driven, so no operator is present to pass `--no-backup`. The posture
|
|
6
|
-
* (fast = skip
|
|
7
|
-
*
|
|
6
|
+
* (fast = skip the pre-deploy backup; safe = back up first) is DERIVED, with
|
|
7
|
+
* this precedence:
|
|
8
8
|
* 1. per-module `upgrade_policy: always-safe` — a hard floor (always safe).
|
|
9
9
|
* 2. per-release `deploy_posture` stamped in the .netapp release metadata.
|
|
10
10
|
* 3. per-module `upgrade_policy: always-fast`.
|
|
11
11
|
* 4. default: the semver delta of installed → next (revision/patch = fast;
|
|
12
12
|
* minor/major = safe).
|
|
13
13
|
*
|
|
14
|
+
* Posture gates the BACKUP ONLY. Both paths run the same post-deploy verify
|
|
15
|
+
* (module-upgrade.ts calls runModuleHealthCheck either way), and neither
|
|
16
|
+
* gates the deploy: verify is the last step, so a failing health check reports
|
|
17
|
+
* "upgraded but verify failed" on an upgrade that has already landed — there
|
|
18
|
+
* is no rollback and no auto-restore. This comment previously claimed safe
|
|
19
|
+
* posture ran a "full verify" against fast's "extended verify"; no such
|
|
20
|
+
* distinction has ever existed in the code, and believing it makes
|
|
21
|
+
* `auto_upgrade` sound far more guarded than it is.
|
|
22
|
+
*
|
|
23
|
+
* `upgrade_policy` (like `auto_upgrade`) is OPERATOR CONFIG, not a manifest
|
|
24
|
+
* key — `celilo module config set <module> upgrade_policy always-safe`. The
|
|
25
|
+
* manifest schema is `.strict()` and declares neither.
|
|
26
|
+
*
|
|
14
27
|
* celilo reads version NUMBERS, never changesets — the default needs only the
|
|
15
28
|
* versions. All functions here are pure (Rule 10).
|
|
16
29
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
-
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
3
3
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
@@ -7,10 +7,11 @@ import { closeDb } from '../db/client';
|
|
|
7
7
|
import { runMigrations } from '../db/migrate';
|
|
8
8
|
import {
|
|
9
9
|
InFlightError,
|
|
10
|
+
OPERATION_TTL_MS,
|
|
10
11
|
checkInFlight,
|
|
11
12
|
completeOperation,
|
|
12
13
|
failOperation,
|
|
13
|
-
|
|
14
|
+
isPidRunnable,
|
|
14
15
|
refuseIfInFlight,
|
|
15
16
|
startOperation,
|
|
16
17
|
} from './module-operations';
|
|
@@ -95,7 +96,7 @@ describe('module-operations', () => {
|
|
|
95
96
|
const child = spawnSync('node', ['-e', 'process.exit(0)']);
|
|
96
97
|
const deadPid = child.pid;
|
|
97
98
|
expect(deadPid).toBeGreaterThan(0);
|
|
98
|
-
expect(
|
|
99
|
+
expect(isPidRunnable(deadPid)).toBe(false);
|
|
99
100
|
|
|
100
101
|
// Insert a fake row with the dead pid via raw SQL (bypasses pid=process.pid in startOperation).
|
|
101
102
|
const { getDb } = require('../db/client');
|
|
@@ -141,14 +142,74 @@ describe('module-operations', () => {
|
|
|
141
142
|
});
|
|
142
143
|
});
|
|
143
144
|
|
|
144
|
-
describe('
|
|
145
|
+
describe('isPidRunnable', () => {
|
|
145
146
|
it('returns true for the current process', () => {
|
|
146
|
-
expect(
|
|
147
|
+
expect(isPidRunnable(process.pid)).toBe(true);
|
|
147
148
|
});
|
|
148
149
|
|
|
149
150
|
it('returns false for a dead pid', () => {
|
|
150
151
|
const child = spawnSync('node', ['-e', 'process.exit(0)']);
|
|
151
|
-
expect(
|
|
152
|
+
expect(isPidRunnable(child.pid as number)).toBe(false);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
// The bug this whole module exists to prevent: a Ctrl-Z'd `module deploy`
|
|
156
|
+
// is still "alive" by kill(pid, 0) and held the backup lock for 20 days.
|
|
157
|
+
it('returns false for a STOPPED process, which kill(pid, 0) calls alive', () => {
|
|
158
|
+
const child = spawn('sleep', ['60'], { stdio: 'ignore' });
|
|
159
|
+
const pid = child.pid as number;
|
|
160
|
+
try {
|
|
161
|
+
expect(isPidRunnable(pid)).toBe(true);
|
|
162
|
+
|
|
163
|
+
child.kill('SIGSTOP');
|
|
164
|
+
// Wait for the state change to land in the process table.
|
|
165
|
+
for (let i = 0; i < 100 && isPidRunnable(pid); i++) spawnSync('sleep', ['0.01']);
|
|
166
|
+
|
|
167
|
+
// Still passes the old liveness test...
|
|
168
|
+
let existsByKill = true;
|
|
169
|
+
try {
|
|
170
|
+
process.kill(pid, 0);
|
|
171
|
+
} catch {
|
|
172
|
+
existsByKill = false;
|
|
173
|
+
}
|
|
174
|
+
expect(existsByKill).toBe(true);
|
|
175
|
+
|
|
176
|
+
// ...but is correctly reported as unable to make progress.
|
|
177
|
+
expect(isPidRunnable(pid)).toBe(false);
|
|
178
|
+
} finally {
|
|
179
|
+
child.kill('SIGCONT');
|
|
180
|
+
child.kill('SIGKILL');
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe('abandonment by age', () => {
|
|
186
|
+
function insertRow(id: string, pid: number, startedAt: Date): void {
|
|
187
|
+
const { getDb } = require('../db/client');
|
|
188
|
+
const { moduleOperations } = require('../db/schema');
|
|
189
|
+
getDb()
|
|
190
|
+
.insert(moduleOperations)
|
|
191
|
+
.values({
|
|
192
|
+
id,
|
|
193
|
+
moduleId: 'ancient',
|
|
194
|
+
operation: 'deploy',
|
|
195
|
+
status: 'in_progress',
|
|
196
|
+
pid,
|
|
197
|
+
startedAt,
|
|
198
|
+
})
|
|
199
|
+
.run();
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
it('ignores a row older than the TTL even though its process is alive', () => {
|
|
203
|
+
// process.pid is unquestionably running, so age is the only thing that
|
|
204
|
+
// can release this row. This is the pid-reuse case: an old row whose
|
|
205
|
+
// number now belongs to some unrelated healthy process.
|
|
206
|
+
insertRow('ancient-row', process.pid, new Date(Date.now() - OPERATION_TTL_MS - 60_000));
|
|
207
|
+
expect(checkInFlight()).toHaveLength(0);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it('still blocks on a young row whose process is alive', () => {
|
|
211
|
+
insertRow('fresh-row', process.pid, new Date(Date.now() - 60_000));
|
|
212
|
+
expect(checkInFlight()).toHaveLength(1);
|
|
152
213
|
});
|
|
153
214
|
});
|
|
154
215
|
});
|
|
@@ -13,13 +13,23 @@
|
|
|
13
13
|
* throw err;
|
|
14
14
|
* }
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* the
|
|
16
|
+
* A row with status='in_progress' stops holding the lock once it looks
|
|
17
|
+
* abandoned, which is three different things:
|
|
18
|
+
*
|
|
19
|
+
* - the process is GONE — it crashed before writing completion
|
|
20
|
+
* - the process is STOPPED — suspended (Ctrl-Z) or a zombie, so it
|
|
21
|
+
* will never reach the completion write
|
|
22
|
+
* - the row is OLD — past `OPERATION_TTL_MS`
|
|
23
|
+
*
|
|
24
|
+
* Only the first was originally handled, and the other two are not
|
|
25
|
+
* hypothetical: a `module deploy` Ctrl-Z'd on a lost terminal held the
|
|
26
|
+
* lock for 20 days and blocked every backup on the fleet.
|
|
27
|
+
*
|
|
28
|
+
* Stale rows are ignored rather than deleted; `celilo module operations`
|
|
29
|
+
* lists them and `... clear` sweeps them.
|
|
21
30
|
*/
|
|
22
31
|
|
|
32
|
+
import { spawnSync } from 'node:child_process';
|
|
23
33
|
import { randomUUID } from 'node:crypto';
|
|
24
34
|
import { eq } from 'drizzle-orm';
|
|
25
35
|
import { getDb } from '../db/client';
|
|
@@ -62,17 +72,51 @@ export function failOperation(operationId: string, error: unknown): void {
|
|
|
62
72
|
}
|
|
63
73
|
|
|
64
74
|
/**
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
75
|
+
* How long an in_progress row may hold the lock before it is treated as
|
|
76
|
+
* abandoned regardless of what its process appears to be doing.
|
|
77
|
+
*
|
|
78
|
+
* This is not belt-and-braces on the liveness check — it is the only
|
|
79
|
+
* check that survives pid reuse. A pid is a recycled number, not a
|
|
80
|
+
* stable identity: a busy host wraps the whole pid space in days, after
|
|
81
|
+
* which an old row's pid names an unrelated live process and the
|
|
82
|
+
* liveness check happily reports "still running" forever. Ageing the row
|
|
83
|
+
* out is the only thing that ends that.
|
|
84
|
+
*
|
|
85
|
+
* Two hours is longer than any real deploy and short enough that a wedge
|
|
86
|
+
* is an inconvenience rather than an outage.
|
|
87
|
+
*/
|
|
88
|
+
export const OPERATION_TTL_MS = 2 * 60 * 60 * 1000;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* True if the process can still make progress on its operation.
|
|
92
|
+
*
|
|
93
|
+
* `kill(pid, 0)` answers only "does this pid exist". A STOPPED process —
|
|
94
|
+
* SIGTSTP from a Ctrl-Z, or a lost controlling terminal — passes that
|
|
95
|
+
* test while being permanently unable to finish, which is exactly how
|
|
96
|
+
* the 20-day wedge happened. `ps -o state=` reports the state itself and
|
|
97
|
+
* is spelled the same on Linux and macOS.
|
|
68
98
|
*/
|
|
69
|
-
export function
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
99
|
+
export function isPidRunnable(pid: number): boolean {
|
|
100
|
+
const result = spawnSync('ps', ['-o', 'state=', '-p', String(pid)], { encoding: 'utf-8' });
|
|
101
|
+
|
|
102
|
+
// No usable `ps`. Fall back to bare existence: a stopped process will
|
|
103
|
+
// still block, which is the old behavior, but we never wrongly release
|
|
104
|
+
// a lock that a live operation is holding.
|
|
105
|
+
if (result.error) {
|
|
106
|
+
try {
|
|
107
|
+
process.kill(pid, 0);
|
|
108
|
+
return true;
|
|
109
|
+
} catch {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
75
112
|
}
|
|
113
|
+
|
|
114
|
+
if (result.status !== 0) return false; // no such process
|
|
115
|
+
|
|
116
|
+
// Linux reports multi-character states ("Tl", "Ss"); the first
|
|
117
|
+
// character is the state proper. T = stopped, Z = zombie.
|
|
118
|
+
const state = result.stdout.trim()[0] ?? '';
|
|
119
|
+
return state !== 'T' && state !== 'Z';
|
|
76
120
|
}
|
|
77
121
|
|
|
78
122
|
export interface InFlightConflict {
|
|
@@ -82,9 +126,10 @@ export interface InFlightConflict {
|
|
|
82
126
|
}
|
|
83
127
|
|
|
84
128
|
/**
|
|
85
|
-
* Returns rows that genuinely look in-flight: status='in_progress'
|
|
86
|
-
*
|
|
87
|
-
*
|
|
129
|
+
* Returns rows that genuinely look in-flight: status='in_progress', the
|
|
130
|
+
* row is younger than `OPERATION_TTL_MS`, AND the originating process is
|
|
131
|
+
* still able to make progress. Everything else is abandoned and excluded,
|
|
132
|
+
* so a crashed, suspended, or forgotten operation cannot wedge the fleet.
|
|
88
133
|
*
|
|
89
134
|
* @param excludeOperationId - operation id to exclude from the check
|
|
90
135
|
* (so an operation doesn't see itself as a conflict).
|
|
@@ -97,10 +142,13 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
|
|
|
97
142
|
.where(eq(moduleOperations.status, 'in_progress'))
|
|
98
143
|
.all();
|
|
99
144
|
|
|
145
|
+
const now = Date.now();
|
|
100
146
|
const conflicts: InFlightConflict[] = [];
|
|
101
147
|
for (const row of rows) {
|
|
102
148
|
if (excludeOperationId && row.id === excludeOperationId) continue;
|
|
103
|
-
|
|
149
|
+
// Age first: it costs nothing, where the liveness probe spawns `ps`.
|
|
150
|
+
if (now - row.startedAt.getTime() > OPERATION_TTL_MS) continue;
|
|
151
|
+
if (!isPidRunnable(row.pid)) continue;
|
|
104
152
|
conflicts.push({
|
|
105
153
|
operation: row,
|
|
106
154
|
describe: `${row.operation} of ${row.moduleId} (pid ${row.pid})`,
|
|
@@ -117,8 +165,10 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
|
|
|
117
165
|
export class InFlightError extends Error {
|
|
118
166
|
constructor(public readonly conflicts: InFlightConflict[]) {
|
|
119
167
|
const list = conflicts.map((c) => ` • ${c.describe}`).join('\n');
|
|
168
|
+
const hint =
|
|
169
|
+
'If it is not really running: "celilo module operations" to inspect, "celilo module operations clear" to release.';
|
|
120
170
|
super(
|
|
121
|
-
`Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run
|
|
171
|
+
`Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.\n${hint}`,
|
|
122
172
|
);
|
|
123
173
|
this.name = 'InFlightError';
|
|
124
174
|
}
|
|
@@ -158,6 +158,34 @@ describe('register / unregister roundtrip', () => {
|
|
|
158
158
|
expect(result.registered).toBe(0);
|
|
159
159
|
});
|
|
160
160
|
|
|
161
|
+
it('a module with an on_backup hook arms the scheduled backup sweep', () => {
|
|
162
|
+
// The sweep is a system-level subscriber, not a module one, so it does not
|
|
163
|
+
// count toward `registered`. It appears the moment the fleet has something
|
|
164
|
+
// to back up — including on `module update`, which is how a manifest that
|
|
165
|
+
// newly declares a cadence reaches an already-deployed fleet.
|
|
166
|
+
const result = registerModuleSubscriptions(
|
|
167
|
+
baseManifest({ hooks: { on_backup: { script: 'backup.ts' } } }),
|
|
168
|
+
'/p',
|
|
169
|
+
);
|
|
170
|
+
expect(result.registered).toBe(0);
|
|
171
|
+
|
|
172
|
+
const bus = openBus({ dbPath, events: defineEvents({}) });
|
|
173
|
+
try {
|
|
174
|
+
const row = bus.db
|
|
175
|
+
.query<{ name: string; pattern: string; handler: string }, []>(
|
|
176
|
+
'SELECT name, pattern, handler FROM subscribers',
|
|
177
|
+
)
|
|
178
|
+
.get();
|
|
179
|
+
expect(row).toEqual({
|
|
180
|
+
name: 'celilo-backup-sweep',
|
|
181
|
+
pattern: 'timer.tick.1h',
|
|
182
|
+
handler: 'celilo backup sweep',
|
|
183
|
+
});
|
|
184
|
+
} finally {
|
|
185
|
+
bus.close();
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
|
|
161
189
|
it('registers each subscription as a row, names scoped to module id', () => {
|
|
162
190
|
const result = registerModuleSubscriptions(
|
|
163
191
|
baseManifest({
|
|
@@ -358,7 +386,10 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
|
|
|
358
386
|
expect(poll).toBeDefined();
|
|
359
387
|
expect(poll?.pattern).toBe('timer.tick.15m');
|
|
360
388
|
// A literal handler command (the CLI poll), not a hook — see manifest comment.
|
|
361
|
-
|
|
389
|
+
// `--poll` is load-bearing: the dispatcher appends the event id to a
|
|
390
|
+
// subprocess handler, and without the flag it lands in the module-name slot
|
|
391
|
+
// ("Module not found: <event_id>" every tick).
|
|
392
|
+
expect(poll?.handler).toBe('celilo module upgrade --poll');
|
|
362
393
|
expect(poll?.hook).toBeUndefined();
|
|
363
394
|
|
|
364
395
|
const resolved = resolveSubscription(
|
|
@@ -368,6 +399,6 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
|
|
|
368
399
|
'/modules/celilo-mgmt',
|
|
369
400
|
);
|
|
370
401
|
expect(resolved.name).toBe('celilo-mgmt.registry-poll');
|
|
371
|
-
expect(resolved.handler).toBe('celilo module upgrade');
|
|
402
|
+
expect(resolved.handler).toBe('celilo module upgrade --poll');
|
|
372
403
|
});
|
|
373
404
|
});
|
|
@@ -21,6 +21,7 @@ import { getEventBusPath, getModuleStoragePath } from '../config/paths';
|
|
|
21
21
|
import { getDb } from '../db/client';
|
|
22
22
|
import { modules } from '../db/schema';
|
|
23
23
|
import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
|
|
24
|
+
import { ensureBackupSweepSubscriber } from './backup-sweep';
|
|
24
25
|
|
|
25
26
|
/**
|
|
26
27
|
* The bus is opened by the celilo CLI without an event registry — the
|
|
@@ -90,10 +91,18 @@ export function registerModuleSubscriptions(
|
|
|
90
91
|
modulePath: string,
|
|
91
92
|
): { registered: number } {
|
|
92
93
|
const subs = manifest.subscriptions ?? [];
|
|
93
|
-
|
|
94
|
+
const backupSweep = Boolean(manifest.hooks?.on_backup);
|
|
95
|
+
if (subs.length === 0 && !backupSweep) return { registered: 0 };
|
|
94
96
|
|
|
95
97
|
const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
96
98
|
try {
|
|
99
|
+
// A module that can be backed up is also what switches the scheduled
|
|
100
|
+
// backup sweep on. Registering here rather than at system init means the
|
|
101
|
+
// sweep appears the moment the fleet has something to back up, and — since
|
|
102
|
+
// `module update` comes through here too — a manifest that newly declares a
|
|
103
|
+
// cadence arms the sweep on the same update that declares it. Idempotent.
|
|
104
|
+
if (backupSweep) ensureBackupSweepSubscriber(bus);
|
|
105
|
+
|
|
97
106
|
for (const sub of subs) {
|
|
98
107
|
const resolved = resolveSubscription(sub, manifest.id, modulePath);
|
|
99
108
|
bus.subscribe(resolved);
|
|
@@ -43,11 +43,30 @@ describe('checkTypeScriptBuild', () => {
|
|
|
43
43
|
}
|
|
44
44
|
});
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
// A tsconfig at the module ROOT is not the one we run: it lives in scripts/,
|
|
47
|
+
// next to the package.json and node_modules that make @celilo/capabilities
|
|
48
|
+
// resolve. Looking at the wrong level is why this check never fired.
|
|
49
|
+
test('ignores a tsconfig at the module root', async () => {
|
|
47
50
|
const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
|
|
48
51
|
try {
|
|
52
|
+
mkdirSync(join(dir, 'scripts'));
|
|
53
|
+
writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
|
|
49
54
|
writeFileSync(join(dir, 'tsconfig.json'), '{}');
|
|
50
55
|
const r = await checkTypeScriptBuild(dir);
|
|
56
|
+
expect(r.status).toBe('warn');
|
|
57
|
+
expect(r.message).toContain('scripts/tsconfig.json');
|
|
58
|
+
} finally {
|
|
59
|
+
rmSync(dir, { recursive: true, force: true });
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('fail with helpful message when tsconfig present but no node_modules', async () => {
|
|
64
|
+
const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
|
|
65
|
+
try {
|
|
66
|
+
mkdirSync(join(dir, 'scripts'));
|
|
67
|
+
writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
|
|
68
|
+
writeFileSync(join(dir, 'scripts', 'tsconfig.json'), '{}');
|
|
69
|
+
const r = await checkTypeScriptBuild(dir);
|
|
51
70
|
expect(r.status).toBe('fail');
|
|
52
71
|
expect(r.message).toContain('node_modules');
|
|
53
72
|
expect(r.message).toContain('bun install');
|