@celilo/cli 2.0.0 → 2.2.0

package/src/registry/client.ts

@@ -11,6 +11,98 @@ import { readFile } from 'node:fs/promises';
 
 export const DEFAULT_REGISTRY = 'https://celilo.computer/registry';
 
+/** Attempts per idempotent registry read, including the first. */
+const GET_ATTEMPTS = 3;
+/** Per-attempt ceiling for metadata reads. Packages use DOWNLOAD_TIMEOUT_MS. */
+const GET_TIMEOUT_MS = 30_000;
+/** Linear backoff: 1s, then 2s. Short, because a deploy is waiting on this. */
+const GET_RETRY_DELAY_MS = 1_000;
+/**
+ * Per-attempt ceiling for a package download.
+ *
+ * This was briefly 180s, on the reasoning that a package is megabytes and
+ * metadata is not. That was wrong, and measurably so: the whole retry budget has
+ * to fit inside the CALLER's window, and celilo's own e2e harness gives a command
+ * 120s by default (`packages/e2e/src/container-manager.ts`). Three attempts at
+ * 180s allows ~543s, so a slow download stopped failing as a download and started
+ * being killed by the caller — a worse error, further from the cause.
+ *
+ * 30s x 3 attempts plus backoff is ~93s, which fits. The durability comes from
+ * retrying, not from a wider window: a 2.6MB transfer that normally takes about a
+ * second does not need 180s, it needs another go.
+ */
+const DOWNLOAD_TIMEOUT_MS = 30_000;
+
+/**
+ * Read a body to completion, reporting progress so a failure can say how far it
+ * got. `Response.arrayBuffer()` gives no partial count when it throws.
+ */
+async function readCounting(
+  resp: Response,
+  onProgress: (bytes: number) => void,
+): Promise<ArrayBuffer> {
+  if (!resp.body) return resp.arrayBuffer();
+
+  const reader = resp.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (value) {
+        chunks.push(value);
+        total += value.byteLength;
+        onProgress(total);
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return out.buffer;
+}
+
+/**
+ * A status worth trying again. 5xx is the server failing, 408 and 429 are it
+ * asking us to wait. Every other 4xx is a settled answer.
+ */
+function isRetryableStatus(status: number): boolean {
+  return status >= 500 || status === 408 || status === 429;
+}
+
+/**
+ * A settled answer from the registry. Retrying cannot change it, so `withRetry`
+ * rethrows it immediately rather than spending the backoff on a known 404.
+ */
+class RegistryAnswer extends Error {}
+
+/** `publish` writes `sha256:<hex>`; tolerate a bare hex digest from older entries. */
+function normalizeCksum(value: string): string {
+  return value.replace(/^sha256:/, '').toLowerCase();
+}
+
+/**
+ * Is this index entry's `cksum` an actual digest we can check against?
+ *
+ * Not every entry carries one. The registry's bootstrap path publishes the
+ * literal string `bootstrap` (packages/registry-server/src/bootstrap.ts), because
+ * those modules are packaged on demand and have no stable digest to publish.
+ * A sentinel means "no integrity data", which is a reason to skip the check and
+ * not a reason to reject the package — treating it as a digest rejects every
+ * bootstrap-served import.
+ */
+function isVerifiableCksum(value: string): boolean {
+  return /^[0-9a-f]{64}$/.test(normalizeCksum(value));
+}
+
 export interface SparseConfig {
   dl: string;
   api: string;
@@ -109,9 +201,56 @@ export class RegistryClient {
     return `${this.baseUrl}/api/v1/modules/${encodeURIComponent(name)}/${encodeURIComponent(version)}/download`;
   }
 
-  async download(name: string, version: string): Promise<ArrayBuffer> {
-    const resp = await this.get(this.downloadUrl(name, version));
-    return resp.arrayBuffer();
+  /**
+   * Fetch a module .netapp, verifying it against the index entry's `cksum`.
+   *
+   * `cksum` has always been computed at publish time and shipped in every index
+   * entry, and nothing on this side ever checked it. A short transfer was
+   * therefore accepted as a complete package and failed later in gunzip as
+   * "zlib: unexpected end of file", which points at the package rather than at
+   * the download that produced it.
+   *
+   * Verification is also what makes the retry mean anything: a fault nobody can
+   * detect is a fault nobody can retry. Pass `expectedCksum` whenever the caller
+   * holds the entry — both callers do.
+   */
+  async download(name: string, version: string, expectedCksum?: string): Promise<ArrayBuffer> {
+    const url = this.downloadUrl(name, version);
+
+    return this.withRetry(async () => {
+      const startedAt = Date.now();
+      const resp = await this.fetchOnce(url, DOWNLOAD_TIMEOUT_MS);
+
+      // Read in chunks so a failure can say how far it got. A bare
+      // `arrayBuffer()` that times out reports nothing, which is why three runs
+      // of this failure told us only that it was slow and never how slow. Bytes
+      // and elapsed together separate a steady trickle from a stall, and those
+      // two want different fixes (a bigger window versus an idle timeout).
+      let data: ArrayBuffer;
+      let received = 0;
+      try {
+        data = await readCounting(resp, (n) => {
+          received = n;
+        });
+      } catch (err) {
+        const seconds = (Date.now() - startedAt) / 1000;
+        const rate = seconds > 0 ? received / 1024 / seconds : 0;
+        throw new Error(
+          `Download of ${name}@${version} failed after ${received} bytes in ${seconds.toFixed(1)}s (${rate.toFixed(0)} KiB/s): ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+
+      if (expectedCksum && isVerifiableCksum(expectedCksum)) {
+        const actual = createHash('sha256').update(new Uint8Array(data)).digest('hex');
+        if (actual !== normalizeCksum(expectedCksum)) {
+          throw new Error(
+            `Package ${name}@${version} failed its integrity check: got ${data.byteLength} bytes with sha256 ${actual}, expected ${normalizeCksum(expectedCksum)}. A short or corrupted download, not a bad package.`,
+          );
+        }
+      }
+
+      return data;
+    });
   }
 
   /**
@@ -232,14 +371,67 @@ export class RegistryClient {
   }
 
   private async get(url: string): Promise<Response> {
-    // 30s (was 15s): a module .netapp can be tens of MB and the download must
-    // finish within one window. 15s was too aggressive on slow links — the
-    // download of a ~19MB module over a multi-hop path (e2e sim NAT, real WAN)
-    // intermittently timed out mid-transfer.
-    const resp = await fetch(url, { signal: AbortSignal.timeout(30_000) });
-    if (!resp.ok) {
-      throw new Error(`Registry error at ${url}: HTTP ${resp.status}`);
+    // 30s per attempt: a module .netapp can be tens of MB, and the download of a
+    // ~19MB module over a multi-hop path (e2e sim NAT, real WAN) intermittently
+    // times out mid-transfer.
+    //
+    // The window used to be the whole story — one attempt, and the response to
+    // observed flakiness was widening it from 15s to 30s. A wider single window
+    // is not a durable fetch, it is a bigger gap to fall through. Measured
+    // 2026-09-04: `module import iptables` failed twice with "Download failed:
+    // The operation timed out" on a loaded host, while the identical call had
+    // succeeded minutes earlier. This client is the fleet's module delivery
+    // path, so a single transient failing an import is an operator-facing
+    // outage, not just an e2e flake.
+    //
+    // Every caller of get() is an idempotent read (config, search, metadata,
+    // download), so retrying is safe. `publish` does not route through here.
+    // A 4xx is an answer and is never retried: a 404 must stay fast and say
+    // "not found" rather than stall for the whole backoff.
+    return this.withRetry(() => this.fetchOnce(url));
+  }
+
+  /** One attempt. Throws `RegistryAnswer` for a status retrying cannot change. */
+  private async fetchOnce(url: string, timeoutMs = GET_TIMEOUT_MS): Promise<Response> {
+    const resp = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+    if (resp.ok) return resp;
+
+    const message = `Registry error at ${url}: HTTP ${resp.status}`;
+    if (!isRetryableStatus(resp.status)) throw new RegistryAnswer(message);
+    throw new Error(message);
+  }
+
+  /**
+   * Retry a whole operation, not a piece of one.
+   *
+   * The unit matters. Retrying only the request leaves the body download
+   * outside the retry, and a body is where a large transfer actually fails.
+   */
+  private async withRetry<T>(run: () => Promise<T>): Promise<T> {
+    const startedAt = Date.now();
+    let lastError: unknown;
+
+    for (let attempt = 1; attempt <= GET_ATTEMPTS; attempt++) {
+      try {
+        return await run();
+      } catch (err) {
+        if (err instanceof RegistryAnswer) throw err;
+        lastError = err;
+        if (attempt < GET_ATTEMPTS) {
+          await new Promise((resolve) => setTimeout(resolve, GET_RETRY_DELAY_MS * attempt));
+        }
+      }
     }
-    return resp;
+
+    // Exhausting the budget is itself the event worth reporting. The bare last
+    // error said only what the final attempt said, which is the same string the
+    // caller got before the retry existed, so the operator cannot tell whether
+    // the retry ran (celilo#1264). The original error rides along as `cause`.
+    const seconds = (Date.now() - startedAt) / 1000;
+    const detail = lastError instanceof Error ? lastError.message : String(lastError);
+    throw new Error(
+      `Registry fetch failed after ${GET_ATTEMPTS} attempts over ${seconds.toFixed(1)}s: ${detail}`,
+      { cause: lastError },
+    );
   }
 }

package/src/services/alerting/coverage-source.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Health-coverage's view of the control plane (celilo#1225).
+ *
+ * `celilo-mgmt` declares no `health_check` hook, and that is deliberate: celilo
+ * runs `runFleetChecks` for it directly, which asks eight questions where the
+ * hook asked two. The hook was deleted because it could not answer them from
+ * inside the jail — it reported a present database missing and failed the
+ * deploy.
+ *
+ * Without the carve-out here, the module best observed by celilo would be the
+ * one celilo warns is unobservable. That warning would be both false and
+ * unfixable, since the fix it names — add a health_check hook — is the thing
+ * that was removed on purpose.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DbClient } from '../../db/client';
+import { modules } from '../../db/schema';
+import { setupTestDatabaseAt } from '../../test-utils/database';
+import { CONTROL_PLANE_MODULE_ID } from '../deployed-systems';
+import { loadModuleCoverage } from './coverage-source';
+import { healthCoverageFailingKeys } from './health-coverage';
+
+describe('health coverage and the control plane', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'cov-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabaseAt(dbPath);
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  function installModule(id: string, manifestData: Record<string, unknown>): void {
+    db.insert(modules)
+      .values({
+        id,
+        name: id,
+        version: '1.0.0',
+        state: 'INSTALLED',
+        sourcePath: `/modules/${id}`,
+        manifestData,
+      })
+      .run();
+  }
+
+  test('the control plane counts as verifiable despite declaring no hook', () => {
+    installModule(CONTROL_PLANE_MODULE_ID, { id: CONTROL_PLANE_MODULE_ID, hooks: {} });
+
+    const coverage = loadModuleCoverage(db);
+    const controlPlane = coverage.find((m) => m.id === CONTROL_PLANE_MODULE_ID);
+
+    expect(controlPlane?.hasHealthCheckHook).toBe(true);
+    // The finding that must NOT be raised — the one whose remedy is to add
+    // back the hook this change deleted.
+    const failing = healthCoverageFailingKeys(coverage);
+    const neverVerified = failing.filter((f) => f.message.includes('can never be verified'));
+    expect(neverVerified).toHaveLength(0);
+  });
+
+  test('an ordinary module with no hook is still reported unobservable', () => {
+    // The carve-out is for the control plane alone. If it leaked to every
+    // module, the check would stop finding anything and look healthy.
+    installModule('homebridge', { id: 'homebridge', hooks: {} });
+
+    const failing = healthCoverageFailingKeys(loadModuleCoverage(db));
+    const messages = failing.map((f) => f.message);
+    expect(
+      messages.some((m) => m.includes('homebridge') && m.includes('can never be verified')),
+    ).toBe(true);
+  });
+});

package/src/services/alerting/coverage-source.ts

@@ -9,6 +9,7 @@
 import type { DbClient } from '../../db/client';
 import { modules } from '../../db/schema';
 import type { ModuleManifest } from '../../manifest/schema';
+import { CONTROL_PLANE_MODULE_ID } from '../deployed-systems';
 import { loadModuleHealthCadences } from './health-cadence';
 import type { ModuleCoverageInput } from './health-coverage';
 
@@ -28,7 +29,16 @@ export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
       return {
         id: module.id,
         state: module.state,
-        hasHealthCheckHook: Boolean(manifest.hooks?.health_check),
+        // The control plane is verifiable without declaring a hook, and that
+        // is the point rather than an exemption: celilo runs `runFleetChecks`
+        // for it directly (`controlPlaneHealthChecks`), which asks eight
+        // questions where its old hook asked two. The hook was deleted because
+        // it could not answer them from inside the jail — it reported a
+        // present database missing (celilo#1225). Reading only the manifest
+        // here would raise a coverage warning about the best-observed module
+        // in the fleet.
+        hasHealthCheckHook:
+          Boolean(manifest.hooks?.health_check) || module.id === CONTROL_PLANE_MODULE_ID,
         cadence: cadences.get(module.id)?.cadence ?? null,
       };
     });

package/src/services/alerting/format.test.ts

@@ -6,9 +6,11 @@ import {
   humaniseRemaining,
   moduleHealthCell,
   renderAlertTable,
+  renderMonitorRunMessage,
   sortAlertRows,
   toAlertRow,
 } from './format';
+import type { MonitorRunOutcomeSummary } from './run-monitor';
 
 const NOW = new Date('2026-07-28T12:00:00Z');
 const ago = (minutes: number) => new Date(NOW.getTime() - minutes * 60_000);
@@ -190,3 +192,58 @@ describe('moduleHealthCell', () => {
     );
   });
 });
+
+describe('renderMonitorRunMessage — names the failing check (#1266)', () => {
+  function summary(over: Partial<MonitorRunOutcomeSummary>): MonitorRunOutcomeSummary {
+    return {
+      monitorId: 'mon-1',
+      outcome: 'success',
+      createdIds: [],
+      resolvedIds: [],
+      ...over,
+    } as MonitorRunOutcomeSummary;
+  }
+
+  const caddyRunning = {
+    key: 'module:caddy/check:caddy_running',
+    severity: 'critical' as const,
+    message: 'caddy systemd service is not active',
+  };
+
+  test('a failing check is named, with its reason', () => {
+    const message = renderMonitorRunMessage('caddy', summary({ failingKeys: [caddyRunning] }));
+    expect(message).toContain('module:caddy/check:caddy_running');
+    expect(message).toContain('caddy systemd service is not active');
+    expect(message).toContain('1 failing');
+  });
+
+  test('every failing key is named, not just the first', () => {
+    const diskSpace = {
+      key: 'module:caddy/check:disk-space',
+      severity: 'warning' as const,
+      message: '/var 94% used',
+    };
+    const message = renderMonitorRunMessage(
+      'caddy',
+      summary({ failingKeys: [caddyRunning, diskSpace] }),
+    );
+    expect(message).toContain('module:caddy/check:caddy_running');
+    expect(message).toContain('module:caddy/check:disk-space');
+    expect(message).toContain('2 failing');
+  });
+
+  test('a healthy run keeps the zero the suite greps for', () => {
+    const message = renderMonitorRunMessage('caddy', summary({ failingKeys: [] }));
+    expect(message).toContain('0 failing');
+    expect(message).toContain('0 resolved');
+  });
+
+  test('a run that could not execute says why instead of inventing a count', () => {
+    const message = renderMonitorRunMessage(
+      'caddy',
+      summary({ outcome: 'error', errorMessage: 'hook exited 1' }),
+    );
+    expect(message).toContain('check could not run');
+    expect(message).toContain('hook exited 1');
+  });
+});

package/src/services/alerting/format.ts

@@ -10,6 +10,7 @@
  */
 
 import type { Alert, AlertState } from '../../db/schema';
+import type { MonitorRunOutcomeSummary } from './run-monitor';
 
 export interface AlertRow {
   key: string;
@@ -148,3 +149,26 @@ export function moduleHealthCell(input: {
   if (input.firingCount > 0) return `${input.firingCount} firing`;
   return 'ok';
 }
+
+/**
+ * The message `celilo monitor run <target>` reports.
+ *
+ * The count line stays greppable (`0 failing` is what a healthy fleet prints),
+ * and each failing key is named underneath it with its reason — an operator
+ * paged by this line learns WHAT failed, not just that something did (#1266).
+ *
+ * Pure — takes the run summary, returns the string — so the wording can be
+ * asserted without a database or a fleet.
+ */
+export function renderMonitorRunMessage(target: string, summary: MonitorRunOutcomeSummary): string {
+  if (summary.outcome === 'error') {
+    return `${target}: check could not run — ${summary.errorMessage ?? 'unknown error'}`;
+  }
+  const lines = [
+    `${target}: ${summary.failingKeys.length} failing, ${summary.resolvedIds.length} resolved`,
+  ];
+  for (const failing of summary.failingKeys) {
+    lines.push(`  [${failing.severity}] ${failing.key}: ${failing.message}`);
+  }
+  return lines.join('\n');
+}

package/src/services/alerting/run-monitor.ts

@@ -47,7 +47,7 @@ export interface MonitorRunOutcomeSummary {
   monitorId: string;
   outcome: 'success' | 'error';
   errorMessage?: string;
-  failingKeyCount: number;
+  failingKeys: FailingKey[];
   createdIds: string[];
   resolvedIds: string[];
 }
@@ -186,7 +186,7 @@ export async function runOneMonitor(
     monitorId: monitor.id,
     outcome: product.outcome,
     errorMessage: product.errorMessage,
-    failingKeyCount: product.failingKeys.length,
+    failingKeys: product.failingKeys,
     createdIds,
     resolvedIds,
   };

package/src/services/backup-create.ts

@@ -6,15 +6,16 @@
 import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
+import { create as tarCreate } from 'tar';
 import { getDbPath, getMasterKeyPath } from '../config/paths';
 import { getDb } from '../db/client';
 import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema';
 import { invokeHook } from '../hooks/executor';
+import { createHookStores } from '../hooks/hook-store';
 import { createConsoleLogger } from '../hooks/logger';
 import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
-import { shellEscape } from '../utils/shell';
 import { encryptFileToFile } from './backup-cipher';
 import { buildManifest } from './backup-manifest';
 import {
@@ -153,8 +154,7 @@ export async function createSystemStateBackup(
     // Tar the envelope. Operator can re-extract a backup file manually
     // for diagnostics: `age -d -p file.backup | tar -t` shows the contents.
     const tarPath = join(tempDir, 'envelope.tar');
-    const { execSync } = await import('node:child_process');
-    execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
+    await tarCreate({ file: tarPath, cwd: envelopeDir }, ['.']);
 
     // Encrypt the tar, streamed — see backup-cipher.ts. The plaintext is
     // never held in memory, so a large fleet DB can't OOM the snapshot.
@@ -401,6 +401,7 @@ export async function createModuleBackup(
         debug: false,
         systems: getModuleSystems(moduleId, db),
         remoteAccess: remoteAccessPolicy(moduleId, db),
+        hookStores: () => createHookStores(db, moduleId),
       },
     );
 
@@ -434,8 +435,7 @@ export async function createModuleBackup(
 
     // Tar the envelope (manifest.json + data/).
     const tarPath = join(tempDir, 'envelope.tar');
-    const { execSync } = await import('node:child_process');
-    execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
+    await tarCreate({ file: tarPath, cwd: envelopeDir }, ['.']);
 
     // Encrypt the tar, streamed — see backup-cipher.ts. Module artifacts run
     // to hundreds of MB (forgejo's are ~774 MB); holding one in memory is
@@ -566,6 +566,7 @@ export async function importModuleBackup(
           debug: false,
           systems: getModuleSystems(moduleId, db),
           remoteAccess: remoteAccessPolicy(moduleId, db),
+          hookStores: () => createHookStores(db, moduleId),
         },
       );
 
@@ -590,8 +591,7 @@ export async function importModuleBackup(
 
     // Tar the artifacts
     const tarPath = join(tempDir, 'backup.tar');
-    const { execSync } = await import('node:child_process');
-    execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(artifactDir)} .`);
+    await tarCreate({ file: tarPath, cwd: artifactDir }, ['.']);
 
     // Encrypt the tar, streamed — see backup-cipher.ts.
     const masterKey = await getOrCreateMasterKey();

package/src/services/backup-envelope-roundtrip.test.ts

@@ -16,16 +16,25 @@
 
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import { execSync } from 'node:child_process';
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  copyFileSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { getDbPath } from '../config/paths';
 import { closeDb, getDb } from '../db/client';
 import { runMigrations } from '../db/migrate';
 import { backups, systemConfig } from '../db/schema';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { decryptFileToFile, encryptFileToFile } from './backup-cipher';
 import { createSystemStateBackup } from './backup-create';
-import { MANIFEST_SCHEMA_VERSION, parseManifest } from './backup-manifest';
+import { MANIFEST_SCHEMA_VERSION, buildManifest, parseManifest } from './backup-manifest';
 import { restoreSystemStateBackup } from './backup-restore';
 import { addBackupStorage, setDefaultBackupStorage, verifyBackupStorage } from './backup-storage';
 
@@ -121,6 +130,40 @@ describe('backup envelope round-trip', () => {
     expect(restoreResult.error).toBeUndefined();
   });
 
+  it('an archive written by SYSTEM tar (the pre-library producer) restores via the tar library', async () => {
+    // The tar library replaced `tar -cf/-xf` shell-outs (celilo#1235). A
+    // self-consistent round-trip cannot detect a layout change, because the
+    // new extract sites would happily read back the new layout. So this test
+    // builds the envelope with the SYSTEM tar — byte-for-byte the command the
+    // old code ran — and restores it with the current code, proving a backup
+    // taken BEFORE the change still restores after it.
+    const db = getDb();
+    db.insert(systemConfig).values({ key: 'pre-change-sentinel', value: 'before-backup' }).run();
+
+    // Take a real backup to get a valid row + storage location, then swap the
+    // artifact for one produced by the system tar the old code shelled out to.
+    const result = await createSystemStateBackup();
+    expect(result.success).toBe(true);
+    const artifactPath = join(storageDir, 'celilo-backups', result.storagePath as string);
+
+    const envelopeDir = join(dir, 'pre-change-envelope');
+    mkdirSync(envelopeDir, { recursive: true });
+    copyFileSync(getDbPath(), join(envelopeDir, 'celilo.db'));
+    writeFileSync(
+      join(envelopeDir, 'manifest.json'),
+      JSON.stringify(buildManifest({ kind: 'system' }), null, 2),
+    );
+    const tarPath = join(dir, 'pre-change.tar');
+    execSync(`tar -cf '${tarPath}' -C '${envelopeDir}' .`);
+    const masterKey = await getOrCreateMasterKey();
+    await encryptFileToFile(tarPath, artifactPath, masterKey);
+
+    const backupRow = db.select().from(backups).all()[0];
+    const restoreResult = await restoreSystemStateBackup(backupRow);
+    expect(restoreResult.success).toBe(true);
+    expect(restoreResult.error).toBeUndefined();
+  });
+
   it('system restore refuses an artifact with a wrong manifest kind', async () => {
     // Create a valid backup, then poison the manifest by re-packing.
     const result = await createSystemStateBackup();

package/src/services/backup-restore.ts

@@ -4,21 +4,21 @@
  * For system state backups, restores the Celilo database.
  */
 
-import { execSync } from 'node:child_process';
 import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
+import { extract as tarExtract } from 'tar';
 import { getDbPath } from '../config/paths';
 import { closeDb, getDb } from '../db/client';
 import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema';
 import type { Backup } from '../db/schema';
 import { invokeHook } from '../hooks/executor';
+import { createHookStores } from '../hooks/hook-store';
 import { createConsoleLogger } from '../hooks/logger';
 import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
-import { shellEscape } from '../utils/shell';
 import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { createStorageProvider } from './backup-storage';
@@ -70,7 +70,7 @@ export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreR
     mkdirSync(envelopeDir, { recursive: true });
     const tarPath = join(tempDir, 'envelope.tar');
     await decryptFileToFile(encryptedPath, tarPath, masterKey);
-    execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
+    await tarExtract({ file: tarPath, cwd: envelopeDir });
 
     // Read + validate manifest BEFORE touching the live DB. An
     // incompatible artifact must not get past this point.
@@ -180,6 +180,9 @@ export async function restoreModuleBackup(
   if (!backup.moduleId) {
     return { success: false, error: 'Backup has no associated module' };
   }
+  // Captured because the narrowing above does not survive into the hookStores
+  // callback below, where TypeScript widens the parameter back to string|null.
+  const moduleId = backup.moduleId;
 
   const db = getDb();
   const mod = db.select().from(modules).where(eq(modules.id, backup.moduleId)).get();
@@ -221,7 +224,7 @@ export async function restoreModuleBackup(
     const masterKey = await getOrCreateMasterKey();
     const tarPath = join(tempDir, 'envelope.tar');
     await decryptFileToFile(encryptedPath, tarPath, masterKey);
-    execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
+    await tarExtract({ file: tarPath, cwd: envelopeDir });
 
     // Read + validate envelope manifest BEFORE invoking the hook.
     const manifestPath = join(envelopeDir, 'manifest.json');
@@ -290,6 +293,7 @@ export async function restoreModuleBackup(
         debug: false,
         systems: getModuleSystems(backup.moduleId, db),
         remoteAccess: remoteAccessPolicy(backup.moduleId, db),
+        hookStores: () => createHookStores(db, moduleId),
       },
     );