@celilo/cli 0.16.1 → 0.17.0

package/src/cli/index.ts

@@ -839,6 +839,9 @@ Subcommands:
   list                  List all configured storage destinations
   verify <storage-id>   Test storage connectivity and permissions
   set-default <id>      Set the default backup storage destination
+  set-path <id> <path>  Relocate a local destination to a new directory
+    Options:
+      --no-migrate      Change the path without moving existing archives
   remove <storage-id>   Remove a storage destination
     Options:
       --force           Skip confirmation prompts
@@ -860,6 +863,10 @@ Examples:
   # Set default destination
   celilo storage set-default local-backups
 
+  # Move a local destination (archives at the old path are moved too;
+  # if that path is gone, the change proceeds with nothing to migrate)
+  celilo storage set-path local-backups /var/lib/celilo/backups
+
   # Remove storage
   celilo storage remove local-backups --force
 
@@ -1836,6 +1843,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
       return handleStorageSetDefault(parsed.args, parsed.flags);
     }
 
+    if (parsed.subcommand === 'set-path') {
+      const { handleStorageSetPath } = await import('./commands/storage-set-path');
+      return handleStorageSetPath(parsed.args, parsed.flags);
+    }
+
     return {
       success: false,
       error: `Unknown storage subcommand: ${parsed.subcommand}\n\nRun "celilo storage --help" for usage`,

package/src/cli/tui/audit-state.test.ts

@@ -1,7 +1,9 @@
 import { describe, expect, test } from 'bun:test';
-import type { DriftFinding, SystemAuditReport } from '../../services/audit/types';
+import type { DriftCategory, DriftFinding, SystemAuditReport } from '../../services/audit/types';
 import {
+  ALL_CATEGORIES,
   type AuditTuiState,
+  CATEGORY_LABELS,
   groupFindings,
   initState,
   reducer,
@@ -244,3 +246,15 @@ describe('selectedFinding', () => {
     expect(selectedFinding(initState(report([])))).toBeNull();
   });
 });
+
+// ALL_CATEGORIES is a plain array, so the type system cannot require every
+// DriftCategory to appear in it — a new category compiles fine and then never
+// shows up in the TUI. CATEGORY_LABELS is a Record and IS exhaustive, so it is
+// the honest source of truth to compare against.
+describe('ALL_CATEGORIES covers every category', () => {
+  test('every labelled category is listed, and vice versa', () => {
+    expect([...ALL_CATEGORIES].sort()).toEqual(
+      (Object.keys(CATEGORY_LABELS) as DriftCategory[]).sort(),
+    );
+  });
+});

package/src/cli/tui/audit-state.ts

@@ -83,6 +83,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
   'secrets_decryptable',
   'services_reachable',
   'machines_reachable',
+  'transport_reads',
   'trusted_sources',
 ];
 
@@ -101,6 +102,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
   secrets_decryptable: 'Secrets',
   services_reachable: 'Service reachability',
   machines_reachable: 'Machine reachability',
+  transport_reads: 'Transport readability',
   trusted_sources: 'Trusted networks',
 };
 

package/src/module/packaging/release-metadata.test.ts

@@ -1,5 +1,14 @@
-import { describe, expect, test } from 'bun:test';
-import { type GitCommandRunner, buildReleaseMetadata, collectGitInfo } from './release-metadata';
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  type GitCommandRunner,
+  buildReleaseMetadata,
+  collectGitInfo,
+  makeRealGitRunner,
+} from './release-metadata';
 
 describe('buildReleaseMetadata', () => {
   test('produces a stable shape from injected inputs', () => {
@@ -77,7 +86,7 @@ describe('collectGitInfo', () => {
     expect(calls).toEqual([
       ['rev-parse', 'HEAD'],
       ['rev-parse', '--abbrev-ref', 'HEAD'],
-      ['status', '--porcelain'],
+      ['status', '--porcelain', '--', '.'],
     ]);
   });
 
@@ -101,3 +110,68 @@ describe('collectGitInfo', () => {
     expect(collectGitInfo('/tmp/x', run).dirty).toBe(true);
   });
 });
+
+/**
+ * Recurrence gate for #544, against a REAL git repo rather than a fake runner.
+ *
+ * The tests above pin the argv `collectGitInfo` constructs. That is what let the
+ * bug live: they asserted which command was built, never what it concluded, so
+ * they stayed green while the function reported the whole repository's dirt for
+ * every module. A fake `GitCommandRunner` cannot catch this at all — the defect
+ * IS git's real scoping behavior, which a stub by definition does not model.
+ *
+ * So this drives the real `makeRealGitRunner()` against a real checkout. Delete
+ * the `-- .` pathspec and the first test here fails; that is the whole point.
+ */
+describe('collectGitInfo scopes dirt to sourceDir (#544, real git)', () => {
+  let repo: string;
+  let moduleDir: string;
+  const git = (args: string[], cwd: string) =>
+    execFileSync('git', args, { cwd, encoding: 'utf-8' });
+
+  beforeEach(() => {
+    repo = mkdtempSync(join(tmpdir(), 'celilo-gitinfo-'));
+    moduleDir = join(repo, 'modules', 'probe');
+    mkdirSync(moduleDir, { recursive: true });
+    writeFileSync(join(moduleDir, 'manifest.yml'), 'id: probe\n');
+    writeFileSync(join(repo, 'bun.lock'), 'lockfile v1\n');
+
+    git(['init', '-q'], repo);
+    git(['config', 'user.email', 'test@celilo.test'], repo);
+    git(['config', 'user.name', 'Test'], repo);
+    git(['add', '-A'], repo);
+    git(['commit', '-qm', 'init'], repo);
+  });
+
+  afterEach(() => rmSync(repo, { recursive: true, force: true }));
+
+  test('a tracked file rewritten OUTSIDE the module does not make it dirty', () => {
+    // Exactly what broke the release: `bun install` rewrites the tracked root
+    // `bun.lock` between module publishes. The module itself is untouched.
+    writeFileSync(join(repo, 'bun.lock'), 'lockfile v1\nrewritten by bun install\n');
+
+    expect(collectGitInfo(moduleDir, makeRealGitRunner()).dirty).toBe(false);
+  });
+
+  test("a sibling module's dirt does not make this module dirty", () => {
+    const sibling = join(repo, 'modules', 'other');
+    mkdirSync(sibling, { recursive: true });
+    writeFileSync(join(sibling, 'manifest.yml'), 'id: other\n');
+
+    expect(collectGitInfo(moduleDir, makeRealGitRunner()).dirty).toBe(false);
+  });
+
+  test("the module's OWN dirt still blocks the publish", () => {
+    // The control. Scoping must not have simply disabled the check — this is
+    // the case the guard exists for, and it must still fire.
+    writeFileSync(join(moduleDir, 'manifest.yml'), 'id: probe\nversion: 9.9.9\n');
+
+    expect(collectGitInfo(moduleDir, makeRealGitRunner()).dirty).toBe(true);
+  });
+
+  test('an untracked file inside the module still blocks the publish', () => {
+    writeFileSync(join(moduleDir, 'stray.txt'), 'oops\n');
+
+    expect(collectGitInfo(moduleDir, makeRealGitRunner()).dirty).toBe(true);
+  });
+});

package/src/module/packaging/release-metadata.ts

@@ -129,7 +129,17 @@ export function collectGitInfo(sourceDir: string, run: GitCommandRunner): GitInf
   // `git status --porcelain` prints one line per modified/untracked file.
   // Empty output = clean. Null (command failure) is treated as not-dirty
   // because we don't want to falsely block a publish.
-  const status = run(['status', '--porcelain'], sourceDir);
+  //
+  // The `-- .` pathspec is load-bearing (#544). `git status` reports the WHOLE
+  // repository regardless of cwd, so passing `sourceDir` as cwd scoped nothing:
+  // every caller asks about one module, and got back the dirt of all of them
+  // plus the repo root. The release pipeline runs `bun install` between module
+  // publishes, which rewrites the tracked `bun.lock` at the root — so modules
+  // 1..N published fine and the next one failed with "Working tree at
+  // modules/<clean-module> has uncommitted changes", naming a directory that
+  // was clean and sending you to inspect it. Order-dependent, so it looked
+  // like a random module failing.
+  const status = run(['status', '--porcelain', '--', '.'], sourceDir);
   const dirty = status !== null && status.length > 0;
 
   return {

package/src/services/alerting/builtin-monitors.ts

@@ -24,6 +24,7 @@ import { type FailingKey, builtinAlertKey } from './keys';
  */
 const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
   machines_reachable: 'machine',
+  transport_reads: 'module',
   services_reachable: 'service',
   services_credentials: 'service',
   backups: 'module',

package/src/services/alerting/inbound-poller.test.ts

@@ -7,7 +7,12 @@ import { eq } from 'drizzle-orm';
 import type { DbClient } from '../../db/client';
 import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
 import { setupTestDatabase } from '../../test-utils/setup-test-db';
-import { type InboundPollDeps, pollInbound, transportsWithRoutes } from './inbound-poller';
+import {
+  type InboundPollDeps,
+  type TransportReadRecord,
+  pollInbound,
+  transportsWithRoutes,
+} from './inbound-poller';
 import { moduleCheckAlertKey } from './keys';
 import { createPerson, createRoute } from './people';
 import { mintDelivery } from './tokens';
@@ -386,6 +391,63 @@ describe('pollInbound', () => {
     expect(ackedBy()).toBe(peterRoute.personId);
   });
 
+  // The gap this closes. The ONLY per-transport state celilo persisted was the
+  // cursor, and writeCursor early-returns when there is no cursor — which a
+  // failed read never produces. So the store could not REPRESENT a failure, and
+  // "unreadable for a week" and "nobody replied for a week" left identical
+  // traces (#501).
+  describe('recording what each read attempt produced', () => {
+    const records: Array<[string, TransportReadRecord]> = [];
+    const recording = (over: Partial<InboundPollDeps> = {}) =>
+      deps([], { recordRead: (t, r) => records.push([t, r]), ...over });
+
+    beforeEach(() => {
+      records.length = 0;
+    });
+
+    test('a FAILED read is recorded — the case the cursor could never express', async () => {
+      await pollInbound(
+        db,
+        recording({ receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
+      );
+      expect(records).toHaveLength(1);
+      expect(records[0][0]).toBe('signal');
+      expect(records[0][1]).toMatchObject({ outcome: 'failed', error: 'connection refused' });
+    });
+
+    test('a successful read is recorded with how many messages it returned', async () => {
+      await pollInbound(
+        db,
+        deps([inbound(PETER, 'hello')], { recordRead: (t, r) => records.push([t, r]) }),
+      );
+      expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 1 });
+    });
+
+    // Zero messages is not failure. Conflating them is the original bug.
+    test('an empty but successful read records received, not failed', async () => {
+      await pollInbound(db, recording());
+      expect(records[0][1]).toMatchObject({ outcome: 'received', messages: 0 });
+    });
+
+    test('a unidirectional transport is recorded as such', async () => {
+      await pollInbound(db, recording({ receiveFrom: async () => ({ status: 'unidirectional' }) }));
+      expect(records[0][1].outcome).toBe('unidirectional');
+    });
+
+    // Every attempt, not just interesting ones — a gap in the record would be
+    // read as "nothing happened".
+    test('every attempt is recorded, and carries when it happened', async () => {
+      await pollInbound(db, recording());
+      expect(records[0][1].at).toBe(NOW.toISOString());
+    });
+
+    // Optional dep: a caller that does not persist must still poll.
+    test('polling works with no recorder attached', async () => {
+      const report = await pollInbound(db, deps([]));
+      expect(report.transportsPolled).toBe(1);
+    });
+  });
+
   test('a route pointing at a transport nobody uses is not polled', () => {
     db.delete(alerts).run();
     expect(transportsWithRoutes(db)).toEqual(['signal']);

package/src/services/alerting/inbound-poller.ts

@@ -44,6 +44,35 @@ export interface TransportFailure {
   error: string;
 }
 
+/**
+ * The outcome of one attempt to read a transport, as recorded for later.
+ *
+ * Written on EVERY attempt, including failures — which is the whole point. The
+ * only per-transport state celilo persisted before this was the cursor, and a
+ * failed read produces no cursor, so `writeCursor` returned early and nothing
+ * was written. The store could not REPRESENT a failure, so an absence of
+ * recorded failures was never evidence there had been none: a transport that
+ * had not been readable for a week looked identical to one nobody had replied
+ * on (#501).
+ */
+export interface TransportReadRecord {
+  /** When the attempt happened, ISO-8601. */
+  at: string;
+  outcome: 'received' | 'unidirectional' | 'failed';
+  /** Present only when `failed`. */
+  error?: string;
+  /** How many messages the read returned. Zero is not the same as failure. */
+  messages: number;
+  /**
+   * When a read last SUCCEEDED, carried forward across failures.
+   *
+   * This is the field that answers the question the old state could not: a
+   * transport reporting "0 messages" for a week and one that has not been
+   * readable for a week are the same picture until you can see this.
+   */
+  lastSuccessAt?: string;
+}
+
 /** A message that was read but not acted on, and why. */
 export interface UnheardMessage {
   senderAddress: string;
@@ -65,6 +94,13 @@ export interface InboundPollDeps {
   /** Persisted receive cursor per transport. */
   readCursor(transportModuleId: string): string | null;
   writeCursor(transportModuleId: string, cursor: string | null): void;
+  /**
+   * Record what one read attempt produced. Called for EVERY attempt — a failed
+   * read must leave a trace, or "we could not read this transport" stays
+   * indistinguishable from "nobody replied". Optional so a caller that does not
+   * care (tests, one-off invocations) need not supply it.
+   */
+  recordRead?(transportModuleId: string, record: TransportReadRecord): void;
   now(): Date;
   /**
    * Transport for a route, so an ack can be broadcast to everyone else paged.
@@ -131,6 +167,12 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
   for (const transportId of transportsWithRoutes(db)) {
     const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
     report.transportsPolled++;
+    deps.recordRead?.(transportId, {
+      at: deps.now().toISOString(),
+      outcome: received.status,
+      ...(received.status === 'failed' ? { error: received.error } : {}),
+      messages: received.status === 'received' ? received.messages.length : 0,
+    });
     // One dead transport must not stop the others being read — but it is
     // RECORDED rather than skipped in silence, because "cannot read" and
     // "nothing to read" are the two things an operator most needs to tell

package/src/services/alerting/read-records.ts

@@ -0,0 +1,85 @@
+/**
+ * The per-transport record of whether celilo can still READ replies.
+ *
+ * Extracted from the poll command because it now has two readers: the poller
+ * writes it, and the audit asserts on it. Keeping the key format private to the
+ * writer would have meant the audit re-deriving a string literal — the sort of
+ * duplication that silently stops matching.
+ *
+ * The shape it stores is the point. celilo's only per-transport state used to
+ * be the receive cursor, and `writeCursor` returns early when there is no
+ * cursor — which a failed read never produces. So the store could not
+ * REPRESENT a failure, and "unreadable since Tuesday" left exactly the same
+ * trace as "nobody replied since Tuesday" (#501).
+ */
+
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../../db/client';
+import { systemConfig } from '../../db/schema';
+import { type TransportReadRecord, transportsWithRoutes } from './inbound-poller';
+
+const READ_PREFIX = 'alerting.last_read.';
+
+export function readLastRead(db: DbClient, transportModuleId: string): TransportReadRecord | null {
+  const row = db
+    .select()
+    .from(systemConfig)
+    .where(eq(systemConfig.key, `${READ_PREFIX}${transportModuleId}`))
+    .get();
+  if (!row?.value) return null;
+  try {
+    return JSON.parse(row.value) as TransportReadRecord;
+  } catch {
+    // A malformed row must not read as "never succeeded" — that would invent a
+    // fact and page someone about it.
+    return null;
+  }
+}
+
+/**
+ * Record one attempt, carrying the last SUCCESS forward.
+ *
+ * Deliberately has no counterpart to `writeCursor`'s early return: a failure
+ * that writes nothing is why this record did not exist before. Carrying the
+ * success forward is what turns a log into an answer — a run of failures must
+ * not erase when the transport last actually worked.
+ */
+export function writeLastRead(
+  db: DbClient,
+  transportModuleId: string,
+  record: TransportReadRecord,
+): void {
+  const key = `${READ_PREFIX}${transportModuleId}`;
+  const previous = readLastRead(db, transportModuleId);
+  const lastSuccessAt =
+    record.outcome === 'received' ? record.at : (previous?.lastSuccessAt ?? undefined);
+  const value = JSON.stringify({ ...record, ...(lastSuccessAt ? { lastSuccessAt } : {}) });
+
+  const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
+  if (existing) {
+    db.update(systemConfig).set({ value }).where(eq(systemConfig.key, key)).run();
+  } else {
+    db.insert(systemConfig)
+      .values({ key, value, description: `Last inbound read attempt for ${transportModuleId}` })
+      .run();
+  }
+}
+
+export interface TransportReadStatus {
+  transportModuleId: string;
+  last: TransportReadRecord | null;
+}
+
+/**
+ * Read state for every transport that has a route pointing at it.
+ *
+ * Scoped to transports with routes on purpose: a transport nobody is routed to
+ * cannot fail to deliver anyone's acknowledgement, and paging about it would be
+ * noise that trains an operator to ignore this check.
+ */
+export function readAllTransportStatuses(db: DbClient): TransportReadStatus[] {
+  return transportsWithRoutes(db).map((transportModuleId) => ({
+    transportModuleId,
+    last: readLastRead(db, transportModuleId),
+  }));
+}

package/src/services/audit/index.test.ts

@@ -32,6 +32,7 @@ const emptyDeps = {
   secretsDecryptable: { results: [] },
   servicesReachable: { results: [] },
   machinesReachable: { results: [] },
+  transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
   trustedSources: { firewalls: [] },
 };
 

package/src/services/audit/index.ts

@@ -25,6 +25,7 @@ import {
 } from './services-credentials';
 import { type ServicesReachableAuditDeps, auditServicesReachable } from './services-reachable';
 import { type TerraformPlanAuditDeps, auditTerraformPlan } from './terraform-plan';
+import { type TransportReadsAuditDeps, auditTransportReads } from './transport-reads';
 import { type TrustedSourcesAuditDeps, auditTrustedSources } from './trusted-sources';
 import {
   type DriftCategory,
@@ -53,6 +54,7 @@ export interface AuditDeps {
   secretsDecryptable: SecretsDecryptableAuditDeps;
   servicesReachable: ServicesReachableAuditDeps;
   machinesReachable: MachinesReachableAuditDeps;
+  transportReads: TransportReadsAuditDeps;
   trustedSources: TrustedSourcesAuditDeps;
   /** Defaults to `Date.now()`-based ISO string. */
   now?: () => Date;
@@ -103,6 +105,7 @@ export async function runAudit(
     wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
     wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
     wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
+    wrap('transport_reads', auditTransportReads(deps.transportReads)),
     wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
   ]);
 

package/src/services/audit/transport-reads.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, test } from 'bun:test';
+import type { TransportReadStatus } from '../alerting/read-records';
+import { auditTransportReads } from './transport-reads';
+
+const NOW = new Date('2026-08-01T12:00:00Z');
+const STALE_AFTER = 30 * 60_000;
+
+const ago = (ms: number) => new Date(NOW.getTime() - ms).toISOString();
+
+function status(over: Partial<TransportReadStatus['last']> | null): TransportReadStatus {
+  return {
+    transportModuleId: 'signal',
+    last:
+      over === null
+        ? null
+        : { at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0), ...over },
+  };
+}
+
+const run = (statuses: TransportReadStatus[]) =>
+  auditTransportReads({ statuses, now: NOW, staleAfterMs: STALE_AFTER });
+
+describe('auditTransportReads', () => {
+  test('a transport read successfully just now is not a finding', async () => {
+    expect(await run([status({})])).toEqual([]);
+  });
+
+  // THE case this check exists for. Six tokens were issued and zero consumed
+  // over a week, and nothing anywhere was red (#501).
+  test('a transport not read successfully for hours is drift', async () => {
+    const findings = await run([
+      status({
+        at: ago(60_000),
+        outcome: 'failed',
+        error: 'refused',
+        lastSuccessAt: ago(3 * 3600_000),
+      }),
+    ]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]).toMatchObject({
+      category: 'transport_reads',
+      code: 'transport_reads_stale',
+      severity: 'drift',
+      subject: 'signal',
+    });
+    expect(findings[0].message).toContain('3h ago');
+  });
+
+  // The decision that makes staleness safe to page on: an EMPTY read is a
+  // success, so a quiet transport keeps refreshing lastSuccessAt. If empty
+  // reads counted as failure this check would page on every quiet afternoon
+  // and be switched off within a week.
+  test('a quiet transport — successful reads, zero messages — is NOT stale', async () => {
+    const findings = await run([
+      status({ at: ago(0), outcome: 'received', messages: 0, lastSuccessAt: ago(0) }),
+    ]);
+    expect(findings).toEqual([]);
+  });
+
+  test('a transport that has never succeeded is drift, even if attempts are recent', async () => {
+    const findings = await run([
+      status({
+        at: ago(0),
+        outcome: 'failed',
+        error: 'connection refused',
+        lastSuccessAt: undefined,
+      }),
+    ]);
+    expect(findings[0]).toMatchObject({ code: 'transport_never_read', severity: 'drift' });
+    expect(findings[0].details).toContain('connection refused');
+  });
+
+  test('a transport with no record at all is drift, not silently fine', async () => {
+    const findings = await run([status(null)]);
+    expect(findings[0]).toMatchObject({ code: 'transport_never_polled', severity: 'drift' });
+  });
+
+  // A transport with no `receive` is working as designed. Paging about it
+  // would be paging about a healthy system, which teaches operators to ignore
+  // the check.
+  test('a unidirectional transport is never a finding', async () => {
+    const findings = await run([
+      status({ at: ago(10 * 3600_000), outcome: 'unidirectional', lastSuccessAt: undefined }),
+    ]);
+    expect(findings).toEqual([]);
+  });
+
+  test('just inside the threshold is not yet drift', async () => {
+    expect(await run([status({ lastSuccessAt: ago(STALE_AFTER - 1_000) })])).toEqual([]);
+  });
+
+  test('just outside the threshold is drift', async () => {
+    const findings = await run([status({ lastSuccessAt: ago(STALE_AFTER + 1_000) })]);
+    expect(findings).toHaveLength(1);
+  });
+
+  // The remediation must point at something that does not consume the queue.
+  // `celilo alerts poll` would READ, and a suggestion that eats the operator's
+  // acknowledgement is worse than no suggestion (#541).
+  test('a stale finding sends you to the journal, not to a read', async () => {
+    const findings = await run([status({ lastSuccessAt: ago(3 * 3600_000) })]);
+    expect(findings[0].remediation).toBe('celilo module journal signal');
+  });
+
+  test('each transport is judged on its own record', async () => {
+    const findings = await run([
+      { transportModuleId: 'signal', last: status({}).last },
+      { transportModuleId: 'sms', last: status({ lastSuccessAt: ago(9 * 3600_000) }).last },
+    ]);
+    expect(findings).toHaveLength(1);
+    expect(findings[0].subject).toBe('sms');
+  });
+});