@celilo/cli 0.14.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +18 -2
  3. package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +3 -3
  6. package/src/cli/commands/alerts-list.ts +10 -0
  7. package/src/cli/commands/alerts-poll.ts +12 -6
  8. package/src/cli/commands/alerts-sweep.ts +22 -81
  9. package/src/cli/commands/backup-sweep.ts +65 -0
  10. package/src/cli/commands/module-operations.test.ts +93 -0
  11. package/src/cli/commands/module-operations.ts +134 -0
  12. package/src/cli/commands/module-upgrade.test.ts +32 -20
  13. package/src/cli/commands/module-upgrade.ts +37 -32
  14. package/src/cli/commands/monitor.ts +26 -6
  15. package/src/cli/commands/system-audit.ts +3 -30
  16. package/src/cli/completion.ts +18 -1
  17. package/src/cli/index.ts +11 -0
  18. package/src/db/schema.ts +5 -3
  19. package/src/manifest/schema.ts +4 -1
  20. package/src/module/packaging/build.ts +4 -0
  21. package/src/services/alerting/builtin-source.ts +17 -2
  22. package/src/services/alerting/delivery-loop.test.ts +5 -1
  23. package/src/services/alerting/format.test.ts +0 -1
  24. package/src/services/alerting/inbound-poller.test.ts +44 -8
  25. package/src/services/alerting/inbound-poller.ts +65 -28
  26. package/src/services/alerting/notify-deps.ts +113 -0
  27. package/src/services/alerting/run-monitor.ts +0 -1
  28. package/src/services/alerting/store.test.ts +1 -1
  29. package/src/services/alerting/store.ts +0 -2
  30. package/src/services/alerting/sweep-runner.test.ts +11 -2
  31. package/src/services/alerting/sweep-runner.ts +14 -7
  32. package/src/services/audit/backup-source.ts +54 -0
  33. package/src/services/audit/backups.test.ts +7 -2
  34. package/src/services/audit/backups.ts +10 -18
  35. package/src/services/backup-cipher.test.ts +188 -0
  36. package/src/services/backup-cipher.ts +178 -0
  37. package/src/services/backup-create.ts +20 -30
  38. package/src/services/backup-envelope-roundtrip.test.ts +6 -26
  39. package/src/services/backup-restore.ts +10 -16
  40. package/src/services/backup-schedule.ts +35 -0
  41. package/src/services/backup-sweep.test.ts +148 -0
  42. package/src/services/backup-sweep.ts +124 -0
  43. package/src/services/deploy-posture.ts +15 -2
  44. package/src/services/module-operations.test.ts +67 -6
  45. package/src/services/module-operations.ts +69 -19
  46. package/src/services/module-subscriptions.test.ts +33 -2
  47. package/src/services/module-subscriptions.ts +10 -1
  48. package/src/services/module-validator/typescript-build.test.ts +20 -1
  49. package/src/services/module-validator/typescript-build.ts +9 -5
  50. package/src/services/restore-from-file.ts +6 -21
  51. package/src/templates/generator.test.ts +88 -0
  52. package/src/templates/generator.ts +119 -16
@@ -0,0 +1,148 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { ModuleManifest } from '../manifest/schema';
3
+ import type { BackupSchedule } from './backup-schedule';
4
+ import {
5
+ BACKUP_SWEEP_PATTERN,
6
+ BACKUP_SWEEP_SUBSCRIBER,
7
+ type BackupSweepDeps,
8
+ type BackupSweepModule,
9
+ ensureBackupSweepSubscriber,
10
+ runBackupSweep,
11
+ } from './backup-sweep';
12
+ import { InFlightError } from './module-operations';
13
+
14
+ function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
15
+ const manifest = {
16
+ id,
17
+ hooks: { on_backup: { script: 'backup.ts' } },
18
+ ...(schedule ? { backup: { schedule } } : {}),
19
+ } as unknown as ModuleManifest;
20
+ return { id, manifest };
21
+ }
22
+
23
+ function deps(
24
+ modules: BackupSweepModule[],
25
+ overrides: Partial<BackupSweepDeps> = {},
26
+ ): BackupSweepDeps & { pruned: string[] } {
27
+ const pruned: string[] = [];
28
+ return {
29
+ pruned,
30
+ listEligible: () => modules,
31
+ isDue: () => true,
32
+ backup: async () => ({ success: true }),
33
+ prune: async ({ id }) => {
34
+ pruned.push(id);
35
+ },
36
+ ...overrides,
37
+ };
38
+ }
39
+
40
+ describe('runBackupSweep', () => {
41
+ test('backs up a module whose declared cadence is due', async () => {
42
+ const d = deps([moduleWith('authentik', 'daily')]);
43
+ const report = await runBackupSweep(d);
44
+
45
+ expect(report.backedUp).toEqual(['authentik']);
46
+ expect(report.failures).toEqual([]);
47
+ expect(d.pruned).toEqual(['authentik']);
48
+ });
49
+
50
+ test('skips a module that is not yet due', async () => {
51
+ const report = await runBackupSweep(
52
+ deps([moduleWith('authentik', 'daily')], { isDue: () => false }),
53
+ );
54
+
55
+ expect(report.backedUp).toEqual([]);
56
+ expect(report.skippedNotDue).toEqual(['authentik']);
57
+ });
58
+
59
+ test('never auto-backs-up an explicit manual schedule', async () => {
60
+ const report = await runBackupSweep(deps([moduleWith('scratch', 'manual')]));
61
+
62
+ expect(report.backedUp).toEqual([]);
63
+ expect(report.skippedManual).toEqual(['scratch']);
64
+ });
65
+
66
+ test('an undeclared schedule is backed up, not treated as manual', async () => {
67
+ // The regression this whole subsystem exists for: forgejo and signal
68
+ // declare no `backup:` block and had never been backed up.
69
+ const report = await runBackupSweep(deps([moduleWith('forgejo')]));
70
+
71
+ expect(report.backedUp).toEqual(['forgejo']);
72
+ expect(report.skippedManual).toEqual([]);
73
+ });
74
+
75
+ test('a held operation lock is a skip, not a failure, and stops the pass', async () => {
76
+ const d = deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
77
+ backup: async () => {
78
+ throw new InFlightError([]);
79
+ },
80
+ });
81
+ const report = await runBackupSweep(d);
82
+
83
+ expect(report.skippedLocked).toEqual(['authentik']);
84
+ expect(report.failures).toEqual([]);
85
+ expect(report.backedUp).toEqual([]);
86
+ expect(d.pruned).toEqual([]);
87
+ });
88
+
89
+ test('a failed backup is recorded and the remaining modules still run', async () => {
90
+ const report = await runBackupSweep(
91
+ deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
92
+ backup: async (moduleId) =>
93
+ moduleId === 'authentik' ? { success: false, error: 'hook exited 1' } : { success: true },
94
+ }),
95
+ );
96
+
97
+ expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'hook exited 1' }]);
98
+ expect(report.backedUp).toEqual(['forgejo']);
99
+ });
100
+
101
+ test('a thrown storage error fails only that module', async () => {
102
+ const report = await runBackupSweep(
103
+ deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
104
+ backup: async (moduleId) => {
105
+ if (moduleId === 'authentik') throw new Error('Storage not verified');
106
+ return { success: true };
107
+ },
108
+ }),
109
+ );
110
+
111
+ expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'Storage not verified' }]);
112
+ expect(report.backedUp).toEqual(['forgejo']);
113
+ });
114
+
115
+ test('does not prune when a backup failed', async () => {
116
+ const d = deps([moduleWith('authentik', 'daily')], {
117
+ backup: async () => ({ success: false, error: 'nope' }),
118
+ });
119
+ await runBackupSweep(d);
120
+
121
+ expect(d.pruned).toEqual([]);
122
+ });
123
+ });
124
+
125
+ describe('ensureBackupSweepSubscriber', () => {
126
+ test('registers the hourly sweep handler', () => {
127
+ const calls: Array<{
128
+ name: string;
129
+ pattern: string;
130
+ handler: string;
131
+ registeredBy?: string;
132
+ }> = [];
133
+ ensureBackupSweepSubscriber({
134
+ subscribe: (options) => calls.push(options),
135
+ });
136
+
137
+ expect(calls).toEqual([
138
+ {
139
+ name: BACKUP_SWEEP_SUBSCRIBER,
140
+ pattern: BACKUP_SWEEP_PATTERN,
141
+ handler: 'celilo backup sweep',
142
+ registeredBy: 'celilo-backup',
143
+ },
144
+ ]);
145
+ // 1h is the coarsest tick that can still serve an `hourly` cadence.
146
+ expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
147
+ });
148
+ });
@@ -0,0 +1,124 @@
1
+ /**
2
+ * The thing that actually runs backups on a schedule.
3
+ *
4
+ * A manifest's `backup.schedule` was decorative until this existed: the drift
5
+ * audit reported staleness and `celilo module backup` needed a human. This is
6
+ * the pass that makes a module declaring `daily` get backed up daily with
7
+ * nobody watching.
8
+ *
9
+ * No new scheduling mechanism. Registered as an ordinary bus subscriber whose
10
+ * handler is `celilo backup sweep` against `timer.tick.1h`, exactly like the
11
+ * alerting sweep (`services/alerting/monitors.ts`) — the timer already exists,
12
+ * already survives restarts, and already has retry and dedup. One hour is the
13
+ * coarsest tick that can still serve an `hourly` cadence; `1d` cannot.
14
+ *
15
+ * Dependencies are injected so the pass itself tests with no database, no
16
+ * storage, and no hook execution.
17
+ */
18
+
19
+ import type { ModuleManifest } from '../manifest/schema';
20
+ import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
21
+ import { InFlightError } from './module-operations';
22
+
23
+ export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
24
+ export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
25
+
26
+ export interface SubscriberRegistrar {
27
+ subscribe(options: {
28
+ name: string;
29
+ pattern: string;
30
+ handler: string;
31
+ registeredBy?: string;
32
+ }): unknown;
33
+ }
34
+
35
+ /**
36
+ * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
37
+ * every module install and update.
38
+ */
39
+ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
40
+ bus.subscribe({
41
+ name: BACKUP_SWEEP_SUBSCRIBER,
42
+ pattern: BACKUP_SWEEP_PATTERN,
43
+ handler: 'celilo backup sweep',
44
+ registeredBy: 'celilo-backup',
45
+ });
46
+ }
47
+
48
+ export interface BackupSweepModule {
49
+ id: string;
50
+ manifest: ModuleManifest;
51
+ }
52
+
53
+ export interface BackupSweepDeps {
54
+ /** Installed modules that declare an `on_backup` hook. */
55
+ listEligible(): BackupSweepModule[];
56
+ isDue(moduleId: string, schedule: BackupSchedule): boolean;
57
+ backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
58
+ /** Apply the module's declared retention. No-op when it declares none. */
59
+ prune(module: BackupSweepModule): Promise<void>;
60
+ }
61
+
62
+ export interface BackupSweepReport {
63
+ backedUp: string[];
64
+ /** Explicit `schedule: manual` — the author opted out. */
65
+ skippedManual: string[];
66
+ skippedNotDue: string[];
67
+ /** Another module operation held the lock. Not a failure; retried next tick. */
68
+ skippedLocked: string[];
69
+ failures: Array<{ moduleId: string; error: string }>;
70
+ }
71
+
72
+ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
73
+ const report: BackupSweepReport = {
74
+ backedUp: [],
75
+ skippedManual: [],
76
+ skippedNotDue: [],
77
+ skippedLocked: [],
78
+ failures: [],
79
+ };
80
+
81
+ for (const module of deps.listEligible()) {
82
+ const schedule = effectiveBackupSchedule(module.manifest);
83
+ if (schedule === 'manual') {
84
+ report.skippedManual.push(module.id);
85
+ continue;
86
+ }
87
+ if (!deps.isDue(module.id, schedule)) {
88
+ report.skippedNotDue.push(module.id);
89
+ continue;
90
+ }
91
+
92
+ let result: { success: boolean; error?: string };
93
+ try {
94
+ result = await deps.backup(module.id);
95
+ } catch (error) {
96
+ // Refusing to run while a deploy/restore is in flight is correct, and a
97
+ // scheduled run must not bypass it. The lock is global, so once it is
98
+ // held every remaining module would refuse identically — stop the pass
99
+ // rather than collect the same refusal N times. The next tick retries.
100
+ if (error instanceof InFlightError) {
101
+ report.skippedLocked.push(module.id);
102
+ break;
103
+ }
104
+ // Everything else — an unverified storage destination, a missing default
105
+ // — throws before a backup row exists. Record it against the module and
106
+ // keep going; one module's failure must not cancel the rest.
107
+ report.failures.push({
108
+ moduleId: module.id,
109
+ error: error instanceof Error ? error.message : String(error),
110
+ });
111
+ continue;
112
+ }
113
+
114
+ if (!result.success) {
115
+ report.failures.push({ moduleId: module.id, error: result.error ?? 'backup failed' });
116
+ continue;
117
+ }
118
+
119
+ report.backedUp.push(module.id);
120
+ await deps.prune(module);
121
+ }
122
+
123
+ return report;
124
+ }
@@ -3,14 +3,27 @@
3
3
  * openspec/changes/build-bus-poll-cd/proposal.md).
4
4
  *
5
5
  * CD is CI-driven, so no operator is present to pass `--no-backup`. The posture
6
- * (fast = skip backup + extended verify; safe = backup + full verify) is
7
- * DERIVED, with this precedence:
6
+ * (fast = skip the pre-deploy backup; safe = back up first) is DERIVED, with
7
+ * this precedence:
8
8
  * 1. per-module `upgrade_policy: always-safe` — a hard floor (always safe).
9
9
  * 2. per-release `deploy_posture` stamped in the .netapp release metadata.
10
10
  * 3. per-module `upgrade_policy: always-fast`.
11
11
  * 4. default: the semver delta of installed → next (revision/patch = fast;
12
12
  * minor/major = safe).
13
13
  *
14
+ * Posture gates the BACKUP ONLY. Both paths run the same post-deploy verify
15
+ * (module-upgrade.ts calls runModuleHealthCheck either way), and neither
16
+ * gates the deploy: verify is the last step, so a failing health check reports
17
+ * "upgraded but verify failed" on an upgrade that has already landed — there
18
+ * is no rollback and no auto-restore. This comment previously claimed safe
19
+ * posture ran a "full verify" against fast's "extended verify"; no such
20
+ * distinction has ever existed in the code, and believing it makes
21
+ * `auto_upgrade` sound far more guarded than it is.
22
+ *
23
+ * `upgrade_policy` (like `auto_upgrade`) is OPERATOR CONFIG, not a manifest
24
+ * key — `celilo module config set <module> upgrade_policy always-safe`. The
25
+ * manifest schema is `.strict()` and declares neither.
26
+ *
14
27
  * celilo reads version NUMBERS, never changesets — the default needs only the
15
28
  * versions. All functions here are pure (Rule 10).
16
29
  */
@@ -1,5 +1,5 @@
1
1
  import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
2
- import { spawnSync } from 'node:child_process';
2
+ import { spawn, spawnSync } from 'node:child_process';
3
3
  import { mkdtempSync, rmSync } from 'node:fs';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
@@ -7,10 +7,11 @@ import { closeDb } from '../db/client';
7
7
  import { runMigrations } from '../db/migrate';
8
8
  import {
9
9
  InFlightError,
10
+ OPERATION_TTL_MS,
10
11
  checkInFlight,
11
12
  completeOperation,
12
13
  failOperation,
13
- isPidAlive,
14
+ isPidRunnable,
14
15
  refuseIfInFlight,
15
16
  startOperation,
16
17
  } from './module-operations';
@@ -95,7 +96,7 @@ describe('module-operations', () => {
95
96
  const child = spawnSync('node', ['-e', 'process.exit(0)']);
96
97
  const deadPid = child.pid;
97
98
  expect(deadPid).toBeGreaterThan(0);
98
- expect(isPidAlive(deadPid)).toBe(false);
99
+ expect(isPidRunnable(deadPid)).toBe(false);
99
100
 
100
101
  // Insert a fake row with the dead pid via raw SQL (bypasses pid=process.pid in startOperation).
101
102
  const { getDb } = require('../db/client');
@@ -141,14 +142,74 @@ describe('module-operations', () => {
141
142
  });
142
143
  });
143
144
 
144
- describe('isPidAlive', () => {
145
+ describe('isPidRunnable', () => {
145
146
  it('returns true for the current process', () => {
146
- expect(isPidAlive(process.pid)).toBe(true);
147
+ expect(isPidRunnable(process.pid)).toBe(true);
147
148
  });
148
149
 
149
150
  it('returns false for a dead pid', () => {
150
151
  const child = spawnSync('node', ['-e', 'process.exit(0)']);
151
- expect(isPidAlive(child.pid as number)).toBe(false);
152
+ expect(isPidRunnable(child.pid as number)).toBe(false);
153
+ });
154
+
155
+ // The bug this whole module exists to prevent: a Ctrl-Z'd `module deploy`
156
+ // is still "alive" by kill(pid, 0) and held the backup lock for 20 days.
157
+ it('returns false for a STOPPED process, which kill(pid, 0) calls alive', () => {
158
+ const child = spawn('sleep', ['60'], { stdio: 'ignore' });
159
+ const pid = child.pid as number;
160
+ try {
161
+ expect(isPidRunnable(pid)).toBe(true);
162
+
163
+ child.kill('SIGSTOP');
164
+ // Wait for the state change to land in the process table.
165
+ for (let i = 0; i < 100 && isPidRunnable(pid); i++) spawnSync('sleep', ['0.01']);
166
+
167
+ // Still passes the old liveness test...
168
+ let existsByKill = true;
169
+ try {
170
+ process.kill(pid, 0);
171
+ } catch {
172
+ existsByKill = false;
173
+ }
174
+ expect(existsByKill).toBe(true);
175
+
176
+ // ...but is correctly reported as unable to make progress.
177
+ expect(isPidRunnable(pid)).toBe(false);
178
+ } finally {
179
+ child.kill('SIGCONT');
180
+ child.kill('SIGKILL');
181
+ }
182
+ });
183
+ });
184
+
185
+ describe('abandonment by age', () => {
186
+ function insertRow(id: string, pid: number, startedAt: Date): void {
187
+ const { getDb } = require('../db/client');
188
+ const { moduleOperations } = require('../db/schema');
189
+ getDb()
190
+ .insert(moduleOperations)
191
+ .values({
192
+ id,
193
+ moduleId: 'ancient',
194
+ operation: 'deploy',
195
+ status: 'in_progress',
196
+ pid,
197
+ startedAt,
198
+ })
199
+ .run();
200
+ }
201
+
202
+ it('ignores a row older than the TTL even though its process is alive', () => {
203
+ // process.pid is unquestionably running, so age is the only thing that
204
+ // can release this row. This is the pid-reuse case: an old row whose
205
+ // number now belongs to some unrelated healthy process.
206
+ insertRow('ancient-row', process.pid, new Date(Date.now() - OPERATION_TTL_MS - 60_000));
207
+ expect(checkInFlight()).toHaveLength(0);
208
+ });
209
+
210
+ it('still blocks on a young row whose process is alive', () => {
211
+ insertRow('fresh-row', process.pid, new Date(Date.now() - 60_000));
212
+ expect(checkInFlight()).toHaveLength(1);
152
213
  });
153
214
  });
154
215
  });
@@ -13,13 +13,23 @@
13
13
  * throw err;
14
14
  * }
15
15
  *
16
- * Rows with status='in_progress' whose pid is no longer alive are
17
- * treated as abandoned (the process crashed before the completion
18
- * update landed) and ignored by `checkInFlight()`. This keeps a single
19
- * Ctrl-C from wedging the system, at the cost of leaving stale rows in
20
- * the table; a future cleanup command can sweep them.
16
+ * A row with status='in_progress' stops holding the lock once it looks
17
+ * abandoned, which is three different things:
18
+ *
19
+ * - the process is GONE — it crashed before writing completion
20
+ * - the process is STOPPED — suspended (Ctrl-Z) or a zombie, so it
21
+ * will never reach the completion write
22
+ * - the row is OLD — past `OPERATION_TTL_MS`
23
+ *
24
+ * Only the first was originally handled, and the other two are not
25
+ * hypothetical: a `module deploy` Ctrl-Z'd on a lost terminal held the
26
+ * lock for 20 days and blocked every backup on the fleet.
27
+ *
28
+ * Stale rows are ignored rather than deleted; `celilo module operations`
29
+ * lists them and `... clear` sweeps them.
21
30
  */
22
31
 
32
+ import { spawnSync } from 'node:child_process';
23
33
  import { randomUUID } from 'node:crypto';
24
34
  import { eq } from 'drizzle-orm';
25
35
  import { getDb } from '../db/client';
@@ -62,17 +72,51 @@ export function failOperation(operationId: string, error: unknown): void {
62
72
  }
63
73
 
64
74
  /**
65
- * True if the OS still has a process with the given pid. `kill(pid, 0)`
66
- * sends no signal but throws ESRCH if the process is gone the standard
67
- * idiom for liveness on POSIX.
75
+ * How long an in_progress row may hold the lock before it is treated as
76
+ * abandoned regardless of what its process appears to be doing.
77
+ *
78
+ * This is not belt-and-braces on the liveness check — it is the only
79
+ * check that survives pid reuse. A pid is a recycled number, not a
80
+ * stable identity: a busy host wraps the whole pid space in days, after
81
+ * which an old row's pid names an unrelated live process and the
82
+ * liveness check happily reports "still running" forever. Ageing the row
83
+ * out is the only thing that ends that.
84
+ *
85
+ * Two hours is longer than any real deploy and short enough that a wedge
86
+ * is an inconvenience rather than an outage.
87
+ */
88
+ export const OPERATION_TTL_MS = 2 * 60 * 60 * 1000;
89
+
90
+ /**
91
+ * True if the process can still make progress on its operation.
92
+ *
93
+ * `kill(pid, 0)` answers only "does this pid exist". A STOPPED process —
94
+ * SIGTSTP from a Ctrl-Z, or a lost controlling terminal — passes that
95
+ * test while being permanently unable to finish, which is exactly how
96
+ * the 20-day wedge happened. `ps -o state=` reports the state itself and
97
+ * is spelled the same on Linux and macOS.
68
98
  */
69
- export function isPidAlive(pid: number): boolean {
70
- try {
71
- process.kill(pid, 0);
72
- return true;
73
- } catch {
74
- return false;
99
+ export function isPidRunnable(pid: number): boolean {
100
+ const result = spawnSync('ps', ['-o', 'state=', '-p', String(pid)], { encoding: 'utf-8' });
101
+
102
+ // No usable `ps`. Fall back to bare existence: a stopped process will
103
+ // still block, which is the old behavior, but we never wrongly release
104
+ // a lock that a live operation is holding.
105
+ if (result.error) {
106
+ try {
107
+ process.kill(pid, 0);
108
+ return true;
109
+ } catch {
110
+ return false;
111
+ }
75
112
  }
113
+
114
+ if (result.status !== 0) return false; // no such process
115
+
116
+ // Linux reports multi-character states ("Tl", "Ss"); the first
117
+ // character is the state proper. T = stopped, Z = zombie.
118
+ const state = result.stdout.trim()[0] ?? '';
119
+ return state !== 'T' && state !== 'Z';
76
120
  }
77
121
 
78
122
  export interface InFlightConflict {
@@ -82,9 +126,10 @@ export interface InFlightConflict {
82
126
  }
83
127
 
84
128
  /**
85
- * Returns rows that genuinely look in-flight: status='in_progress' AND
86
- * the originating process is still alive. Abandoned rows (process gone)
87
- * are excluded so a stale Ctrl-C doesn't block future operations.
129
+ * Returns rows that genuinely look in-flight: status='in_progress', the
130
+ * row is younger than `OPERATION_TTL_MS`, AND the originating process is
131
+ * still able to make progress. Everything else is abandoned and excluded,
132
+ * so a crashed, suspended, or forgotten operation cannot wedge the fleet.
88
133
  *
89
134
  * @param excludeOperationId - operation id to exclude from the check
90
135
  * (so an operation doesn't see itself as a conflict).
@@ -97,10 +142,13 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
97
142
  .where(eq(moduleOperations.status, 'in_progress'))
98
143
  .all();
99
144
 
145
+ const now = Date.now();
100
146
  const conflicts: InFlightConflict[] = [];
101
147
  for (const row of rows) {
102
148
  if (excludeOperationId && row.id === excludeOperationId) continue;
103
- if (!isPidAlive(row.pid)) continue;
149
+ // Age first: it costs nothing, where the liveness probe spawns `ps`.
150
+ if (now - row.startedAt.getTime() > OPERATION_TTL_MS) continue;
151
+ if (!isPidRunnable(row.pid)) continue;
104
152
  conflicts.push({
105
153
  operation: row,
106
154
  describe: `${row.operation} of ${row.moduleId} (pid ${row.pid})`,
@@ -117,8 +165,10 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
117
165
  export class InFlightError extends Error {
118
166
  constructor(public readonly conflicts: InFlightConflict[]) {
119
167
  const list = conflicts.map((c) => ` • ${c.describe}`).join('\n');
168
+ const hint =
169
+ 'If it is not really running: "celilo module operations" to inspect, "celilo module operations clear" to release.';
120
170
  super(
121
- `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.`,
171
+ `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.\n${hint}`,
122
172
  );
123
173
  this.name = 'InFlightError';
124
174
  }
@@ -158,6 +158,34 @@ describe('register / unregister roundtrip', () => {
158
158
  expect(result.registered).toBe(0);
159
159
  });
160
160
 
161
+ it('a module with an on_backup hook arms the scheduled backup sweep', () => {
162
+ // The sweep is a system-level subscriber, not a module one, so it does not
163
+ // count toward `registered`. It appears the moment the fleet has something
164
+ // to back up — including on `module update`, which is how a manifest that
165
+ // newly declares a cadence reaches an already-deployed fleet.
166
+ const result = registerModuleSubscriptions(
167
+ baseManifest({ hooks: { on_backup: { script: 'backup.ts' } } }),
168
+ '/p',
169
+ );
170
+ expect(result.registered).toBe(0);
171
+
172
+ const bus = openBus({ dbPath, events: defineEvents({}) });
173
+ try {
174
+ const row = bus.db
175
+ .query<{ name: string; pattern: string; handler: string }, []>(
176
+ 'SELECT name, pattern, handler FROM subscribers',
177
+ )
178
+ .get();
179
+ expect(row).toEqual({
180
+ name: 'celilo-backup-sweep',
181
+ pattern: 'timer.tick.1h',
182
+ handler: 'celilo backup sweep',
183
+ });
184
+ } finally {
185
+ bus.close();
186
+ }
187
+ });
188
+
161
189
  it('registers each subscription as a row, names scoped to module id', () => {
162
190
  const result = registerModuleSubscriptions(
163
191
  baseManifest({
@@ -358,7 +386,10 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
358
386
  expect(poll).toBeDefined();
359
387
  expect(poll?.pattern).toBe('timer.tick.15m');
360
388
  // A literal handler command (the CLI poll), not a hook — see manifest comment.
361
- expect(poll?.handler).toBe('celilo module upgrade');
389
+ // `--poll` is load-bearing: the dispatcher appends the event id to a
390
+ // subprocess handler, and without the flag it lands in the module-name slot
391
+ // ("Module not found: <event_id>" every tick).
392
+ expect(poll?.handler).toBe('celilo module upgrade --poll');
362
393
  expect(poll?.hook).toBeUndefined();
363
394
 
364
395
  const resolved = resolveSubscription(
@@ -368,6 +399,6 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
368
399
  '/modules/celilo-mgmt',
369
400
  );
370
401
  expect(resolved.name).toBe('celilo-mgmt.registry-poll');
371
- expect(resolved.handler).toBe('celilo module upgrade');
402
+ expect(resolved.handler).toBe('celilo module upgrade --poll');
372
403
  });
373
404
  });
@@ -21,6 +21,7 @@ import { getEventBusPath, getModuleStoragePath } from '../config/paths';
21
21
  import { getDb } from '../db/client';
22
22
  import { modules } from '../db/schema';
23
23
  import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
24
+ import { ensureBackupSweepSubscriber } from './backup-sweep';
24
25
 
25
26
  /**
26
27
  * The bus is opened by the celilo CLI without an event registry — the
@@ -90,10 +91,18 @@ export function registerModuleSubscriptions(
90
91
  modulePath: string,
91
92
  ): { registered: number } {
92
93
  const subs = manifest.subscriptions ?? [];
93
- if (subs.length === 0) return { registered: 0 };
94
+ const backupSweep = Boolean(manifest.hooks?.on_backup);
95
+ if (subs.length === 0 && !backupSweep) return { registered: 0 };
94
96
 
95
97
  const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
96
98
  try {
99
+ // A module that can be backed up is also what switches the scheduled
100
+ // backup sweep on. Registering here rather than at system init means the
101
+ // sweep appears the moment the fleet has something to back up, and — since
102
+ // `module update` comes through here too — a manifest that newly declares a
103
+ // cadence arms the sweep on the same update that declares it. Idempotent.
104
+ if (backupSweep) ensureBackupSweepSubscriber(bus);
105
+
97
106
  for (const sub of subs) {
98
107
  const resolved = resolveSubscription(sub, manifest.id, modulePath);
99
108
  bus.subscribe(resolved);
@@ -43,11 +43,30 @@ describe('checkTypeScriptBuild', () => {
43
43
  }
44
44
  });
45
45
 
46
- test('fail with helpful message when tsconfig present but no node_modules', async () => {
46
+ // A tsconfig at the module ROOT is not the one we run: it lives in scripts/,
47
+ // next to the package.json and node_modules that make @celilo/capabilities
48
+ // resolve. Looking at the wrong level is why this check never fired.
49
+ test('ignores a tsconfig at the module root', async () => {
47
50
  const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
48
51
  try {
52
+ mkdirSync(join(dir, 'scripts'));
53
+ writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
49
54
  writeFileSync(join(dir, 'tsconfig.json'), '{}');
50
55
  const r = await checkTypeScriptBuild(dir);
56
+ expect(r.status).toBe('warn');
57
+ expect(r.message).toContain('scripts/tsconfig.json');
58
+ } finally {
59
+ rmSync(dir, { recursive: true, force: true });
60
+ }
61
+ });
62
+
63
+ test('fail with helpful message when tsconfig present but no node_modules', async () => {
64
+ const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
65
+ try {
66
+ mkdirSync(join(dir, 'scripts'));
67
+ writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
68
+ writeFileSync(join(dir, 'scripts', 'tsconfig.json'), '{}');
69
+ const r = await checkTypeScriptBuild(dir);
51
70
  expect(r.status).toBe('fail');
52
71
  expect(r.message).toContain('node_modules');
53
72
  expect(r.message).toContain('bun install');