@celilo/cli 0.14.2 → 0.15.0

package/src/services/alerting/sweep-runner.ts

@@ -63,13 +63,16 @@ export interface SweepReport {
   failed: number;
   /**
    * Live alerts nobody is configured to be told about — no escalation policy on
-   * the monitor, so `notifyDepsFor` returns null.
+   * the monitor, so `notifyDepsFor` returns null. Each carries the monitor that
+   * owns it, which is the thing an operator has to assign a policy TO.
    *
-   * Counted rather than skipped in silence: "nothing needed sending" and "an
-   * alert is firing and no policy points at anyone" are opposite situations that
-   * previously rendered identically as `0 notified`.
+   * Identified rather than merely counted: `1 no-policy` told an operator that
+   * something was unreachable but not WHICH thing, so the remedy the sweep
+   * printed alongside it could not be aimed at anything. Assigning the policy to
+   * all eighteen monitors then changed nothing observable (#481). A bare count
+   * is only half a step better than the silence it replaced.
    */
-  noPolicy: number;
+  noPolicy: { alertKey: string; monitor: string }[];
   /**
    * Deliveries escalation declined, keyed by its reason (`within_grace`,
    * `no_eligible_route`, …).
@@ -113,7 +116,7 @@ export async function runSweep(
     deferred: 0,
     deferredDelivered: 0,
     failed: 0,
-    noPolicy: 0,
+    noPolicy: [],
     skipped: {},
     failures: [],
   };
@@ -206,10 +209,14 @@ export async function runSweep(
   }
 
   // 5. Notify. Re-read: the steps above changed state under us.
+  const monitorTargets = new Map(monitors.map((m) => [m.id, m.target]));
   for (const alert of loadAllLiveAlerts(db)) {
     const notifyDeps = deps.notifyDepsFor(alert);
     if (!notifyDeps) {
-      report.noPolicy++;
+      report.noPolicy.push({
+        alertKey: alert.key,
+        monitor: monitorTargets.get(alert.monitorId) ?? '(unknown monitor)',
+      });
       continue;
     }
 

package/src/services/audit/backup-source.ts

@@ -0,0 +1,54 @@
+/**
+ * Reading the roster the backup-freshness check runs against.
+ *
+ * Split out from the check itself so the decision — is this backup too
+ * old — stays pure, while the queries that feed it live here (Rule 2.3).
+ * Shared by `celilo system audit` and by the scheduled `backups` monitor,
+ * so both judge the same fleet from the same data.
+ */
+
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { backups, modules } from '../../db/schema';
+import type { ModuleManifest } from '../../manifest/schema';
+import type { InstalledModuleBackupInfo } from './backups';
+
+const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
+
+/**
+ * Most recent COMPLETED backup per module, in epoch ms.
+ *
+ * The `backups` table is added via inline ALTER statements in
+ * db/client.ts for upgraded databases, so a freshly-initialized DB built
+ * from the drizzle journal alone may not have it yet. Absence means "no
+ * backups recorded", never a crashed audit.
+ */
+function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
+  const latest = new Map<string, number>();
+  try {
+    for (const backup of db.select().from(backups).where(eq(backups.status, 'completed')).all()) {
+      if (!backup.moduleId || !backup.completedAt) continue;
+      const at = backup.completedAt.getTime();
+      const previous = latest.get(backup.moduleId);
+      if (previous === undefined || at > previous) latest.set(backup.moduleId, at);
+    }
+  } catch {
+    // Table missing — leave the map empty.
+  }
+  return latest;
+}
+
+export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
+  const latest = latestSuccessfulBackupByModule(db);
+  return db
+    .select()
+    .from(modules)
+    .all()
+    .filter((module) => DEPLOYED_STATES.includes(module.state))
+    .map((module) => ({
+      id: module.id,
+      state: module.state,
+      manifest: module.manifestData as ModuleManifest,
+      lastSuccessfulBackupAt: latest.get(module.id) ?? null,
+    }));
+}

package/src/services/audit/backups.test.ts

@@ -75,7 +75,10 @@ describe('auditBackups', () => {
     expect(result).toEqual([]);
   });
 
-  test('no schedule declared → treated as manual (no stale flag)', async () => {
+  // Silence-by-default is the bug: an undeclared cadence is how celilo-mgmt
+  // went 55 days without a backup and nobody was told. Opting out takes an
+  // explicit `manual` — see services/backup-schedule.ts.
+  test('no schedule declared → daily, so a year-old backup is stale', async () => {
     const result = await auditBackups({
       modules: [
         makeModule('lunacycle', {
@@ -85,7 +88,9 @@ describe('auditBackups', () => {
       ],
       now: () => NOW,
     });
-    expect(result).toEqual([]);
+    expect(result).toHaveLength(1);
+    expect(result[0]).toMatchObject({ code: 'backup_stale', subject: 'lunacycle' });
+    expect(result[0].message).toContain('daily');
   });
 
   test('daily schedule: 26h-old is stale', async () => {

package/src/services/audit/backups.ts

@@ -8,14 +8,17 @@
  * scheduled run doesn't flag drift on every audit.
  *
  * Modules without an `on_backup` hook are skipped — there's nothing
- * to back up. Modules whose schedule is `manual` (or unset) skip the
- * staleness check (the user decides cadence) but still get a
- * `backup_missing` finding if no backup has ever been recorded.
+ * to back up. Modules whose schedule is explicitly `manual` skip the
+ * staleness check (the operator decides cadence) but still get a
+ * `backup_missing` finding if no backup has ever been recorded. An
+ * unset schedule is `daily`, not `manual` — see
+ * [[services/backup-schedule.ts]] for why that default matters.
  *
  * Time is injected so tests can pin "now" deterministically.
  */
 
 import type { ModuleManifest } from '../../manifest/schema';
+import { effectiveBackupSchedule } from '../backup-schedule';
 import type { DriftFinding } from './types';
 
 export interface InstalledModuleBackupInfo {
@@ -50,10 +53,7 @@ const DAY = 24 * HOUR;
  * - daily   → 25h  (24h + 1h grace)
  * - weekly  → 8d   (7d + 1d grace)
  * - monthly → 32d  (~30d + 2d grace)
- * - manual  → null (no staleness check; user-driven cadence)
- *
- * `undefined` (no `backup:` block in manifest) is treated as
- * `manual` — author opted out of declaring a cadence.
+ * - manual  → null (no staleness check; operator-driven cadence)
  */
 const SCHEDULE_THRESHOLDS = {
   hourly: 2 * HOUR,
@@ -63,21 +63,13 @@ const SCHEDULE_THRESHOLDS = {
   manual: null,
 } as const;
 
-type ScheduleKey = keyof typeof SCHEDULE_THRESHOLDS;
-
 function moduleHasBackupHook(manifest: ModuleManifest): boolean {
   return Boolean(manifest.hooks?.on_backup);
 }
 
-function scheduleFor(manifest: ModuleManifest): ScheduleKey {
-  const s = manifest.backup?.schedule;
-  if (s === 'hourly' || s === 'daily' || s === 'weekly' || s === 'monthly') return s;
-  return 'manual';
-}
-
 function thresholdFor(manifest: ModuleManifest, override: number | undefined): number | null {
   if (override !== undefined) return override;
-  return SCHEDULE_THRESHOLDS[scheduleFor(manifest)];
+  return SCHEDULE_THRESHOLDS[effectiveBackupSchedule(manifest)];
 }
 
 function formatAge(ms: number): string {
@@ -109,7 +101,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
         category: 'backups',
         severity: 'drift',
         code: 'backup_missing',
-        message: `${m.id}: no successful backup recorded (schedule: ${scheduleFor(m.manifest)})`,
+        message: `${m.id}: no successful backup recorded (schedule: ${effectiveBackupSchedule(m.manifest)})`,
         remediation: `celilo backup create ${m.id} --force`,
         actionable: true,
         subject: m.id,
@@ -126,7 +118,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
         category: 'backups',
         severity: 'drift',
         code: 'backup_stale',
-        message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${scheduleFor(m.manifest)}, threshold: ${formatAge(threshold)})`,
+        message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${effectiveBackupSchedule(m.manifest)}, threshold: ${formatAge(threshold)})`,
         remediation: `celilo backup create ${m.id} --force`,
         actionable: true,
         subject: m.id,

package/src/services/backup-cipher.test.ts

@@ -0,0 +1,188 @@
+/**
+ * Backup artifact encryption.
+ *
+ * The load-bearing test here is `writes a streamed artifact, not a JSON
+ * envelope`. Everything else could pass while someone quietly reintroduces
+ * the whole-file-in-memory path — the format assertion is what goes red if
+ * they do, without needing an 800 MB fixture to prove it.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { randomBytes } from 'node:crypto';
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { encryptSecret } from '../secrets/encryption';
+import {
+  ARTIFACT_FORMAT_VERSION,
+  ARTIFACT_MAGIC,
+  decryptFileToFile,
+  encryptFileToFile,
+  isStreamedArtifact,
+} from './backup-cipher';
+
+const MASTER_KEY = Buffer.alloc(32, 7);
+const OTHER_KEY = Buffer.alloc(32, 9);
+
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'celilo-cipher-test-'));
+});
+
+afterEach(() => {
+  rmSync(dir, { recursive: true, force: true });
+});
+
+function paths(name: string) {
+  return {
+    plain: join(dir, `${name}.tar`),
+    enc: join(dir, `${name}.enc`),
+    out: join(dir, `${name}.out`),
+  };
+}
+
+/** An artifact in the pre-streaming format: JSON envelope of base64-of-hex. */
+function writeLegacyArtifact(path: string, payload: Buffer): void {
+  writeFileSync(path, JSON.stringify(encryptSecret(payload.toString('base64'), MASTER_KEY)));
+}
+
+describe('backup-cipher', () => {
+  test('round-trips a payload byte-for-byte', async () => {
+    const p = paths('roundtrip');
+    const payload = randomBytes(3 * 1024 * 1024);
+    writeFileSync(p.plain, payload);
+
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+    await decryptFileToFile(p.enc, p.out, MASTER_KEY);
+
+    expect(readFileSync(p.out).equals(payload)).toBe(true);
+  });
+
+  test('writes a streamed artifact, not a JSON envelope', async () => {
+    const p = paths('format');
+    writeFileSync(p.plain, randomBytes(4096));
+
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    const written = readFileSync(p.enc);
+    expect(written.subarray(0, ARTIFACT_MAGIC.length).equals(ARTIFACT_MAGIC)).toBe(true);
+    expect(written[ARTIFACT_MAGIC.length]).toBe(ARTIFACT_FORMAT_VERSION);
+    expect(isStreamedArtifact(p.enc)).toBe(true);
+
+    // The old path produced base64-of-hex wrapped in JSON, which is 2.67x
+    // the input and starts with '{'. Both are what made it OOM.
+    expect(written[0]).not.toBe('{'.charCodeAt(0));
+    expect(written.length).toBeLessThan(4096 * 2);
+  });
+
+  test('artifact is barely larger than its plaintext', async () => {
+    const p = paths('overhead');
+    const size = 1024 * 1024;
+    writeFileSync(p.plain, randomBytes(size));
+
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    // header (25) + tag (16). AES-GCM is a stream cipher — no block padding.
+    expect(statSync(p.enc).size).toBe(size + 41);
+  });
+
+  test('still reads artifacts in the legacy JSON format', async () => {
+    const p = paths('legacy');
+    const payload = randomBytes(64 * 1024);
+    writeLegacyArtifact(p.enc, payload);
+
+    expect(isStreamedArtifact(p.enc)).toBe(false);
+    await decryptFileToFile(p.enc, p.out, MASTER_KEY);
+
+    expect(readFileSync(p.out).equals(payload)).toBe(true);
+  });
+
+  test('rejects a wrong master key rather than emitting garbage', async () => {
+    const p = paths('wrongkey');
+    writeFileSync(p.plain, randomBytes(8192));
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    expect(decryptFileToFile(p.enc, p.out, OTHER_KEY)).rejects.toThrow();
+  });
+
+  test('rejects tampered ciphertext', async () => {
+    const p = paths('tamper');
+    writeFileSync(p.plain, randomBytes(8192));
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    const bytes = readFileSync(p.enc);
+    bytes[100] ^= 0xff;
+    writeFileSync(p.enc, bytes);
+
+    expect(decryptFileToFile(p.enc, p.out, MASTER_KEY)).rejects.toThrow();
+  });
+
+  test('rejects a tampered auth tag', async () => {
+    const p = paths('tamper-tag');
+    writeFileSync(p.plain, randomBytes(8192));
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    const bytes = readFileSync(p.enc);
+    bytes[bytes.length - 1] ^= 0xff;
+    writeFileSync(p.enc, bytes);
+
+    expect(decryptFileToFile(p.enc, p.out, MASTER_KEY)).rejects.toThrow();
+  });
+
+  test('reports a truncated artifact clearly', async () => {
+    const p = paths('truncated');
+    writeFileSync(p.enc, Buffer.concat([ARTIFACT_MAGIC, Buffer.from([ARTIFACT_FORMAT_VERSION])]));
+
+    expect(decryptFileToFile(p.enc, p.out, MASTER_KEY)).rejects.toThrow(/truncated/i);
+  });
+
+  test('refuses an artifact written by a newer celilo', async () => {
+    const p = paths('future');
+    writeFileSync(p.plain, randomBytes(1024));
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+
+    const bytes = readFileSync(p.enc);
+    bytes[ARTIFACT_MAGIC.length] = ARTIFACT_FORMAT_VERSION + 1;
+    writeFileSync(p.enc, bytes);
+
+    expect(decryptFileToFile(p.enc, p.out, MASTER_KEY)).rejects.toThrow(/Upgrade celilo/);
+  });
+
+  test('handles an empty plaintext', async () => {
+    const p = paths('empty');
+    writeFileSync(p.plain, Buffer.alloc(0));
+
+    await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+    // Header + tag only — nothing to stream back, and a real tar is never
+    // empty, so this is reported rather than silently producing 0 bytes.
+    expect(decryptFileToFile(p.enc, p.out, MASTER_KEY)).rejects.toThrow(/no data/i);
+  });
+
+  /**
+   * The ceiling the old format could not clear: hex-of-base64 is 2.67 chars
+   * per input byte against a ~2^31 max string length, so anything over
+   * ~805 MB was unrepresentable regardless of available RAM.
+   *
+   * Opt-in — it writes ~900 MB to disk and takes tens of seconds, which does
+   * not belong in every CI run. Run deliberately after touching this file:
+   *   CELILO_TEST_LARGE_BACKUP=1 bun test src/services/backup-cipher.test.ts
+   */
+  test.skipIf(!process.env.CELILO_TEST_LARGE_BACKUP)(
+    'round-trips an artifact past the old format ceiling',
+    async () => {
+      const p = paths('huge');
+      const chunk = randomBytes(1024 * 1024);
+      const file = Bun.file(p.plain).writer();
+      for (let i = 0; i < 900; i++) file.write(chunk);
+      await file.end();
+
+      await encryptFileToFile(p.plain, p.enc, MASTER_KEY);
+      await decryptFileToFile(p.enc, p.out, MASTER_KEY);
+
+      expect(existsSync(p.out)).toBe(true);
+      expect(statSync(p.out).size).toBe(statSync(p.plain).size);
+    },
+    300_000,
+  );
+});

package/src/services/backup-cipher.ts

@@ -0,0 +1,178 @@
+/**
+ * Encryption for backup artifacts — file in, file out, streamed.
+ *
+ * Deliberately NOT `encryptSecret`/`decryptSecret`. Those are string-in,
+ * string-out and correct for what they were built for: short values headed
+ * for a DB column. Backup artifacts are the opposite shape, and running them
+ * through a string API cost three full in-memory copies with an expansion
+ * factor at each step:
+ *
+ *   read the tar        774 MB Buffer
+ *   .toString('base64') 1032 MB string
+ *   encrypt to hex      2064 MB string   (hex is 2 chars per byte)
+ *   JSON.stringify      2064 MB string
+ *
+ * — about 6.9 GB live for one 774 MB module, which is what OOM-killed the
+ * forgejo backup. Worse, it had a ceiling no amount of RAM could raise:
+ * hex-of-base64 is 2.67 chars per input byte against a ~2^31 max string
+ * length, so the old path simply could not represent a tar over ~805 MB.
+ *
+ * Here the plaintext never exists in memory at all. `createCipheriv` is a
+ * Transform, so file → cipher → file runs in constant memory regardless of
+ * artifact size, and both intermediate encodings disappear (the ciphertext
+ * is written as raw bytes, so an artifact is now *smaller* than its tar
+ * rather than 2.67x larger).
+ *
+ * On-disk format, all binary:
+ *
+ *   magic      8 bytes   "CELILOBK"
+ *   version    1 byte    currently 1
+ *   iv        16 bytes
+ *   ciphertext  ...      streamed
+ *   auth tag  16 bytes   trailer — GCM only produces it after final()
+ *
+ * The tag has to be a trailer because it does not exist until the last byte
+ * has been encrypted, and seeking back to patch a header would mean the
+ * writer could no longer be a plain stream. Reading it costs one 16-byte
+ * positional read before the stream starts.
+ *
+ * `decryptFileToFile` also reads the previous format (a JSON envelope of
+ * base64-of-hex). The magic bytes are the discriminator: the old writer
+ * always emitted JSON, so a file starting with `{` is legacy. The envelope's
+ * own schemaVersion cannot serve — it lives *inside* the encrypted tar and
+ * is unreadable until after decryption.
+ */
+
+import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
+import {
+  appendFileSync,
+  closeSync,
+  createReadStream,
+  createWriteStream,
+  openSync,
+  readFileSync,
+  readSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { decryptSecret } from '../secrets/encryption';
+import { EncryptionEnvelopeSchema, parseJsonWithValidation } from '../validation/schemas';
+
+const ALGORITHM = 'aes-256-gcm';
+
+/** Identifies a streamed artifact. Legacy artifacts begin with `{`. */
+export const ARTIFACT_MAGIC = Buffer.from('CELILOBK', 'ascii');
+
+/** Bumped only for an incompatible layout change; readers reject unknown values. */
+export const ARTIFACT_FORMAT_VERSION = 1;
+
+const IV_LENGTH = 16;
+const AUTH_TAG_LENGTH = 16;
+const HEADER_LENGTH = ARTIFACT_MAGIC.length + 1 + IV_LENGTH;
+
+/** Read `length` bytes at `offset` without opening a stream. */
+function readBytesAt(path: string, offset: number, length: number): Buffer {
+  const buffer = Buffer.alloc(length);
+  const fd = openSync(path, 'r');
+  try {
+    readSync(fd, buffer, 0, length, offset);
+  } finally {
+    closeSync(fd);
+  }
+  return buffer;
+}
+
+/** Whether this artifact uses the streamed format rather than the JSON envelope. */
+export function isStreamedArtifact(path: string): boolean {
+  if (statSync(path).size < ARTIFACT_MAGIC.length) return false;
+  return readBytesAt(path, 0, ARTIFACT_MAGIC.length).equals(ARTIFACT_MAGIC);
+}
+
+/**
+ * Encrypt `srcPath` to `destPath` in constant memory.
+ */
+export async function encryptFileToFile(
+  srcPath: string,
+  destPath: string,
+  masterKey: Buffer,
+): Promise<void> {
+  const iv = randomBytes(IV_LENGTH);
+  const cipher = createCipheriv(ALGORITHM, masterKey, iv);
+
+  const out = createWriteStream(destPath);
+  out.write(Buffer.concat([ARTIFACT_MAGIC, Buffer.from([ARTIFACT_FORMAT_VERSION]), iv]));
+  await pipeline(createReadStream(srcPath), cipher, out);
+
+  // Available only once the stream has run final(), i.e. after the pipeline
+  // resolves. Appending 16 bytes is O(1) and keeps the writer a plain stream.
+  appendFileSync(destPath, cipher.getAuthTag());
+}
+
+/**
+ * Decrypt `srcPath` to `destPath`. Handles both the streamed format and the
+ * legacy JSON envelope.
+ *
+ * Throws on a truncated artifact, an unknown format version, or a failed
+ * authentication tag (wrong master key, or tampered/corrupted ciphertext).
+ */
+export async function decryptFileToFile(
+  srcPath: string,
+  destPath: string,
+  masterKey: Buffer,
+): Promise<void> {
+  if (!isStreamedArtifact(srcPath)) {
+    decryptLegacyArtifact(srcPath, destPath, masterKey);
+    return;
+  }
+
+  const size = statSync(srcPath).size;
+  const overhead = HEADER_LENGTH + AUTH_TAG_LENGTH;
+  if (size < overhead) {
+    throw new Error(
+      `Backup artifact is truncated: ${size} bytes, but the format needs at least ${overhead}.`,
+    );
+  }
+  if (size === overhead) {
+    throw new Error('Backup artifact contains no data (header and auth tag only).');
+  }
+
+  const header = readBytesAt(srcPath, 0, HEADER_LENGTH);
+  const version = header[ARTIFACT_MAGIC.length];
+  if (version !== ARTIFACT_FORMAT_VERSION) {
+    throw new Error(
+      `Backup artifact uses format version ${version}, but this celilo understands ${ARTIFACT_FORMAT_VERSION}. Upgrade celilo to restore it.`,
+    );
+  }
+
+  const decipher = createDecipheriv(
+    ALGORITHM,
+    masterKey,
+    header.subarray(ARTIFACT_MAGIC.length + 1),
+  );
+  decipher.setAuthTag(readBytesAt(srcPath, size - AUTH_TAG_LENGTH, AUTH_TAG_LENGTH));
+
+  // `end` is inclusive, so the last ciphertext byte is the one before the tag.
+  await pipeline(
+    createReadStream(srcPath, { start: HEADER_LENGTH, end: size - AUTH_TAG_LENGTH - 1 }),
+    decipher,
+    createWriteStream(destPath),
+  );
+}
+
+/**
+ * Read an artifact written before the streamed format.
+ *
+ * Reads the whole thing into memory, which is fine precisely because these
+ * are the artifacts the old writer produced: it could not emit one much over
+ * ~805 MB without dying, so the bound this function relies on is the same bug
+ * that motivated the new format. New artifacts never take this path.
+ */
+function decryptLegacyArtifact(srcPath: string, destPath: string, masterKey: Buffer): void {
+  const envelope = parseJsonWithValidation(
+    readFileSync(srcPath, 'utf-8'),
+    EncryptionEnvelopeSchema,
+    'backup artifact envelope',
+  );
+  writeFileSync(destPath, Buffer.from(decryptSecret(envelope, masterKey), 'base64'));
+}