@celilo/cli 1.0.0 → 1.2.0

package/src/hooks/artifact-retention.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Retention is by AGE, and the test that matters is the one asserting the
+ * FIRST failing run survives a long streak. A count-based rule passes every
+ * other test here and fails that one — which is exactly the bug this
+ * policy exists to avoid, since the first artifact set carries the original
+ * cause and later ones repeat it.
+ *
+ * Uses a real temp directory rather than an injected filesystem: mtime
+ * ordering and directory sizing are the behaviour under test, and a fake
+ * would be asserting my own model of them.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, readdirSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ARTIFACT_RETENTION_MS, pruneModuleArtifacts } from './artifact-retention';
+
+const HOUR = 60 * 60 * 1000;
+const NOW = Date.UTC(2026, 7, 18, 12, 0, 0);
+
+let root: string;
+
+beforeEach(() => {
+  root = mkdtempSync(join(tmpdir(), 'celilo-artifacts-'));
+});
+
+afterEach(() => {
+  rmSync(root, { recursive: true, force: true });
+});
+
+/** Create a run directory whose files are `ageMs` old, with `bytes` of content. */
+function makeRun(name: string, ageMs: number, bytes = 16): string {
+  const dir = join(root, name);
+  mkdirSync(dir, { recursive: true });
+  const file = join(dir, 'spa-failure.png');
+  writeFileSync(file, Buffer.alloc(bytes));
+  const when = new Date(NOW - ageMs);
+  utimesSync(file, when, when);
+  utimesSync(dir, when, when);
+  return dir;
+}
+
+function survivors(): string[] {
+  return readdirSync(root).sort();
+}
+
+describe('pruneModuleArtifacts', () => {
+  test('keeps everything inside the retention window', () => {
+    makeRun('run-a', 1 * HOUR);
+    makeRun('run-b', 23 * HOUR);
+
+    const outcome = pruneModuleArtifacts(root, { now: NOW });
+
+    expect(outcome.removed).toEqual([]);
+    expect(survivors()).toEqual(['run-a', 'run-b']);
+  });
+
+  test('prunes what is older than the retention window', () => {
+    makeRun('stale', 25 * HOUR);
+    makeRun('fresh', 1 * HOUR);
+
+    pruneModuleArtifacts(root, { now: NOW });
+
+    expect(survivors()).toEqual(['fresh']);
+  });
+
+  test('the boundary is exclusive: exactly at the window, it stays', () => {
+    // The off-by-one is the whole risk in a rule expressed as an inequality.
+    makeRun('exactly-24h', ARTIFACT_RETENTION_MS);
+    makeRun('a-ms-older', ARTIFACT_RETENTION_MS + 1);
+
+    pruneModuleArtifacts(root, { now: NOW });
+
+    expect(survivors()).toEqual(['exactly-24h']);
+  });
+
+  test('THE POINT: the first failure of a long streak survives', () => {
+    // A 12-hour streak at the 15-minute health cadence — every run failing,
+    // every one writing artifacts. The first set carries the original cause;
+    // "keep the last N" would have discarded it hours ago.
+    for (let quarterHour = 0; quarterHour <= 48; quarterHour++) {
+      makeRun(`run-${String(quarterHour).padStart(3, '0')}`, quarterHour * 15 * 60 * 1000);
+    }
+
+    pruneModuleArtifacts(root, { now: NOW });
+
+    const kept = survivors();
+    expect(kept).toContain('run-048'); // the oldest — the first failure
+    expect(kept).toContain('run-000'); // the most recent
+    expect(kept.length).toBe(49);
+  });
+
+  test('the size ceiling evicts oldest-first, and only down to the ceiling', () => {
+    makeRun('oldest', 3 * HOUR, 1000);
+    makeRun('middle', 2 * HOUR, 1000);
+    makeRun('newest', 1 * HOUR, 1000);
+
+    const outcome = pruneModuleArtifacts(root, { now: NOW, sizeCeilingBytes: 2500 });
+
+    // 3000 bytes exceeds 2500; dropping the oldest brings it to 2000.
+    expect(survivors()).toEqual(['middle', 'newest']);
+    expect(outcome.removed).toEqual([join(root, 'oldest')]);
+    expect(outcome.retainedBytes).toBe(2000);
+  });
+
+  test('a run still being written is aged by its newest file, not the directory', () => {
+    // The directory's own mtime can lag a file rewritten in place, which
+    // would let an in-flight run read as old enough to evict.
+    const dir = join(root, 'in-flight');
+    mkdirSync(dir, { recursive: true });
+    const old = new Date(NOW - 30 * HOUR);
+    utimesSync(dir, old, old);
+    const stale = join(dir, 'old.txt');
+    writeFileSync(stale, 'x');
+    utimesSync(stale, old, old);
+    writeFileSync(join(dir, 'just-written.png'), 'y'); // now
+
+    pruneModuleArtifacts(root, { now: NOW });
+
+    expect(survivors()).toEqual(['in-flight']);
+  });
+
+  test('a missing artifact root is not an error', () => {
+    expect(() => pruneModuleArtifacts(join(root, 'never-created'), { now: NOW })).not.toThrow();
+  });
+
+  test('loose files beside the run directories are left alone', () => {
+    writeFileSync(join(root, 'cookies.json'), '{}');
+    makeRun('stale', 30 * HOUR);
+
+    pruneModuleArtifacts(root, { now: NOW });
+
+    expect(survivors()).toEqual(['cookies.json']);
+  });
+});

package/src/hooks/artifact-retention.ts

@@ -0,0 +1,159 @@
+/**
+ * Retention for the per-run artifact directories hooks write into.
+ *
+ * **By AGE, not by run count**, and the difference is not cosmetic. For a
+ * persistent failure — the normal case, since a broken deploy stays broken
+ * — the FIRST artifact set carries the original cause and runs 2..N are the
+ * same wall re-hit. A count-based rule therefore keeps the least
+ * informative sets and discards the one worth having: at the 15-minute
+ * cadence a health monitor runs on, "keep the last 5" is seventy-five
+ * minutes, and the operator typically reads the alert hours later.
+ *
+ * The size ceiling is the backstop, not the policy. ~96 runs a day of
+ * full-page screenshots on a management host is a `disk_space` alert
+ * waiting to happen, and celilo has a builtin check that would fire on it.
+ *
+ * Pruning is a pure function of mtime, so it needs no streak-tracking
+ * state and cannot drift out of sync with what is on disk.
+ */
+
+import { readdirSync, rmSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** How long a run's artifacts are kept. Long enough to survive a night. */
+export const ARTIFACT_RETENTION_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Total bytes of retained artifacts per module. Generous — this is the
+ * backstop against an unforeseen writer, not the mechanism that normally
+ * reclaims space.
+ */
+export const ARTIFACT_SIZE_CEILING_BYTES = 64 * 1024 * 1024;
+
+export interface PruneOptions {
+  /** Defaults to `Date.now()`; injected so tests need no sleeping. */
+  now?: number;
+  retentionMs?: number;
+  sizeCeilingBytes?: number;
+}
+
+export interface PruneOutcome {
+  /** Directories removed, oldest first. */
+  removed: string[];
+  /** Bytes retained after pruning. */
+  retainedBytes: number;
+}
+
+interface RunDirectory {
+  path: string;
+  mtimeMs: number;
+  bytes: number;
+}
+
+/**
+ * Remove aged-out and over-ceiling run directories under a module's
+ * artifact root.
+ *
+ * Best effort by construction: a hook run must never fail because a stale
+ * directory could not be deleted, so every filesystem error is swallowed
+ * and the outcome reports only what actually happened.
+ */
+export function pruneModuleArtifacts(
+  artifactRoot: string,
+  options: PruneOptions = {},
+): PruneOutcome {
+  const now = options.now ?? Date.now();
+  const retentionMs = options.retentionMs ?? ARTIFACT_RETENTION_MS;
+  const ceiling = options.sizeCeilingBytes ?? ARTIFACT_SIZE_CEILING_BYTES;
+
+  const runs = readRunDirectories(artifactRoot);
+  // Oldest first, so both passes evict from the same end.
+  runs.sort((a, b) => a.mtimeMs - b.mtimeMs);
+
+  const removed: string[] = [];
+  const surviving: RunDirectory[] = [];
+  for (const run of runs) {
+    if (now - run.mtimeMs > retentionMs) {
+      if (remove(run.path)) removed.push(run.path);
+      continue;
+    }
+    surviving.push(run);
+  }
+
+  let retainedBytes = surviving.reduce((sum, run) => sum + run.bytes, 0);
+  while (retainedBytes > ceiling && surviving.length > 0) {
+    // biome-ignore lint/style/noNonNullAssertion: length checked above
+    const oldest = surviving.shift()!;
+    if (remove(oldest.path)) {
+      removed.push(oldest.path);
+      retainedBytes -= oldest.bytes;
+    } else {
+      // Undeletable: stop rather than spin, and leave it counted.
+      break;
+    }
+  }
+
+  return { removed, retainedBytes };
+}
+
+function readRunDirectories(artifactRoot: string): RunDirectory[] {
+  let entries: string[];
+  try {
+    entries = readdirSync(artifactRoot);
+  } catch {
+    return []; // No artifact root yet — nothing to prune.
+  }
+
+  const runs: RunDirectory[] = [];
+  for (const entry of entries) {
+    const path = join(artifactRoot, entry);
+    try {
+      if (!statSync(path).isDirectory()) continue;
+      runs.push({ path, mtimeMs: newestMtime(path), bytes: directoryBytes(path) });
+    } catch {
+      // Vanished mid-scan, or unreadable. Not ours to fix.
+    }
+  }
+  return runs;
+}
+
+/**
+ * Age a run by the NEWEST file in it, not by the directory's own mtime.
+ * A directory's mtime tracks its last entry change, which on some
+ * filesystems does not move when a file inside it is rewritten in place —
+ * so a run still being appended to could otherwise read as old enough to
+ * evict while the check that owns it is still running.
+ */
+function newestMtime(dir: string): number {
+  let newest = statSync(dir).mtimeMs;
+  for (const file of readdirSync(dir)) {
+    try {
+      newest = Math.max(newest, statSync(join(dir, file)).mtimeMs);
+    } catch {
+      // Skip what we cannot stat.
+    }
+  }
+  return newest;
+}
+
+function directoryBytes(dir: string): number {
+  let total = 0;
+  for (const file of readdirSync(dir)) {
+    try {
+      const stat = statSync(join(dir, file));
+      if (stat.isFile()) total += stat.size;
+    } catch {
+      // Skip what we cannot stat.
+    }
+  }
+  return total;
+}
+
+function remove(path: string): boolean {
+  try {
+    rmSync(path, { recursive: true, force: true });
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/hooks/executor.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, test } from 'bun:test';
+import { existsSync, readdirSync, rmSync } from 'node:fs';
 import { join } from 'node:path';
 import type { ContractHookSignature } from '../manifest/contracts';
 import {
@@ -272,6 +273,85 @@ describe('Hook Executor', () => {
     });
   });
 
+  describe('invokeHook artifacts', () => {
+    test('collects EVERY file the hook wrote, not just the newest .png', async () => {
+      const { logger } = createCapturingLogger();
+      const result = await invokeHook(
+        __dirname,
+        'container_created',
+        '1.0',
+        { script: './test-fixtures/artifact-writing-hook.ts', timeout: 10000 },
+        { vps_ip: '10.0.0.5' },
+        {},
+        {},
+        logger,
+      );
+
+      expect(result.success).toBe(true);
+      const names = (result.artifactPaths ?? []).map((p) => p.split('/').pop());
+      // The old behaviour returned exactly one of these — the .png — and
+      // threw away the DOM and the request log, which is the least useful
+      // third of a post-mortem on its own.
+      expect(names).toEqual(['spa-failure.html', 'spa-failure.png', 'spa-failure.requests.txt']);
+
+      // Per-run, so a consumer writing FIXED filenames cannot overwrite its
+      // own previous run — which is what makes retention meaningful.
+      const dir = (result.artifactPaths ?? [])[0];
+      expect(dir).toContain('/screenshots/container_created-');
+
+      rmSync(join(__dirname, 'screenshots'), { recursive: true, force: true });
+    });
+
+    test('a hook that writes nothing leaves no directory behind', async () => {
+      // Every hook invocation creates a run directory. If empty ones were
+      // kept, a module would accrue one per run forever — ~96 a day for a
+      // 15-minute monitor — and nothing would ever reclaim them.
+      const { logger } = createCapturingLogger();
+      const result = await invokeHook(
+        __dirname,
+        'container_created',
+        '1.0',
+        { script: './test-fixtures/success-hook.ts', timeout: 10000 },
+        { vps_ip: '10.0.0.5' },
+        {},
+        {},
+        logger,
+      );
+
+      expect(result.success).toBe(true);
+      expect(result.artifactPaths).toBeUndefined();
+      expect(existsSync(join(__dirname, 'screenshots', 'container_created'))).toBe(false);
+      const runDirs = existsSync(join(__dirname, 'screenshots'))
+        ? readdirSync(join(__dirname, 'screenshots'))
+        : [];
+      expect(runDirs).toEqual([]);
+    });
+
+    test('an early return before execution leaks no directory', async () => {
+      // The capability pre-flight returns between "create the directory" and
+      // the try/finally that reclaims it. Creating the directory too early
+      // therefore leaked one on every such run, and nothing cleans them up.
+      const { logger } = createCapturingLogger();
+      const result = await invokeHook(
+        __dirname,
+        'container_created',
+        '1.0',
+        { script: './test-fixtures/success-hook.ts', timeout: 10000 },
+        { vps_ip: '10.0.0.5' },
+        {},
+        {},
+        logger,
+        { requiredCapabilities: ['dns_internal'], capabilities: {} },
+      );
+
+      expect(result.success).toBe(false);
+      const runDirs = existsSync(join(__dirname, 'screenshots'))
+        ? readdirSync(join(__dirname, 'screenshots'))
+        : [];
+      expect(runDirs).toEqual([]);
+    });
+  });
+
   describe('invokeHook', () => {
     test('full successful invocation', async () => {
       const { logger, messages } = createCapturingLogger();

package/src/hooks/executor.ts

@@ -19,12 +19,13 @@
  * Execution function (Rule 10.1) - performs side effects (script execution)
  */
 
-import { existsSync, mkdirSync, readdirSync, statSync } from 'node:fs';
-import { join, resolve } from 'node:path';
+import { existsSync, mkdirSync, readdirSync, rmdirSync, statSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
 import {
   type DeployedSystem,
   isCompiledHook,
   isMissingProviderInputError,
+  moduleArtifactDir,
 } from '@celilo/capabilities';
 import {
   type ContractHookSignature,
@@ -33,6 +34,7 @@ import {
   supportedContractVersions,
 } from '../manifest/contracts';
 import { isPrivilegedCapability } from '../manifest/validate';
+import { pruneModuleArtifacts } from './artifact-retention';
 import type { HookContext, HookDefinition, HookLogger, HookResult } from './types';
 
 /** Default total timeout: 60 seconds */
@@ -373,22 +375,45 @@ export function checkRequiredCapabilities(
  * @param since - Only consider files created after this timestamp (ms)
  * @returns Path to screenshot, or undefined
  */
-function findRecentScreenshot(dir: string, since: number): string | undefined {
-  if (!existsSync(dir)) return undefined;
+/**
+ * Every file the hook wrote to this run's artifact directory.
+ *
+ * No mtime filter is needed and none is wanted: the directory is created
+ * fresh for this run, so everything in it was written by this run. The
+ * previous version returned only the newest `.png`, which discarded the
+ * page content and the observed-request log — the two things that make a
+ * screenshot diagnosable.
+ */
+function collectArtifacts(dir: string): string[] {
+  try {
+    return readdirSync(dir)
+      .map((file) => join(dir, file))
+      .filter((path) => {
+        try {
+          return statSync(path).isFile();
+        } catch {
+          return false;
+        }
+      })
+      .sort();
+  } catch {
+    // Best effort — never mask the original error with a readdir failure.
+    return [];
+  }
+}
 
+/** `undefined` rather than `[]`, so an empty result carries no field at all. */
+function nonEmpty(paths: string[]): string[] | undefined {
+  return paths.length > 0 ? paths : undefined;
+}
+
+/** Remove a run's artifact directory when the hook wrote nothing into it. */
+function discardIfEmpty(dir: string): void {
   try {
-    const files = readdirSync(dir).filter((f) => f.endsWith('.png'));
-    for (const file of files) {
-      const fullPath = join(dir, file);
-      const stat = statSync(fullPath);
-      if (stat.mtimeMs >= since) {
-        return fullPath;
-      }
-    }
+    if (readdirSync(dir).length === 0) rmdirSync(dir);
   } catch {
-    // Best effort — don't mask the original error
+    // Best effort — a leftover directory is not worth failing a hook over.
   }
-  return undefined;
 }
 
 /**
@@ -470,9 +495,16 @@ export async function invokeHook(
     };
   }
 
-  // Prepare screenshot directory
-  const screenshotDir = join(modulePath, 'screenshots');
-  mkdirSync(screenshotDir, { recursive: true });
+  // Prepare this run's artifact directory. Per-run, because a hook writing
+  // fixed filenames — a reasonable thing to do — would otherwise overwrite
+  // its own previous artifacts and leave retention nothing to retain.
+  // `moduleArtifactDir` is the one definition of that layout; a bus
+  // subscriber, which gets no HookContext, calls it directly.
+  // Path now, directory later. Creating it here would leak an empty
+  // directory on every early return between this point and the try/finally
+  // below — the capability pre-flight is one — and nothing ever cleans
+  // those up, so a module accrues one per affected run forever.
+  const screenshotDir = moduleArtifactDir(modulePath, `${hookName}-${startTime}`);
 
   // Build context
   const loadedCapabilities = options.capabilities ?? {};
@@ -515,6 +547,13 @@ export async function invokeHook(
     debug,
   );
 
+  // Create the artifact directory only once every early return is behind
+  // us, so the `finally` below is guaranteed to run and reclaim it.
+  mkdirSync(screenshotDir, { recursive: true });
+  // Prune on write, so retention needs no scheduler of its own and cannot
+  // fall behind a module that runs often.
+  pruneModuleArtifacts(dirname(screenshotDir));
+
   // Execute
   try {
     logger.info(`Executing hook: ${hookName}`);
@@ -535,16 +574,17 @@ export async function invokeHook(
     return {
       success: true,
       outputs,
+      artifactPaths: nonEmpty(collectArtifacts(screenshotDir)),
       duration: Date.now() - startTime,
     };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     logger.error(`Hook ${hookName} failed: ${message}`);
 
-    // Check for screenshots captured by the hook
-    const screenshotPath = findRecentScreenshot(screenshotDir, startTime);
-    if (screenshotPath) {
-      logger.info(`Screenshot saved: ${screenshotPath}`);
+    // Artifacts the hook wrote before it failed — the post-mortem.
+    const artifactPaths = nonEmpty(collectArtifacts(screenshotDir));
+    if (artifactPaths) {
+      logger.info(`Artifacts saved:\n  ${artifactPaths.join('\n  ')}`);
     }
 
     // Recognise the cross-module structured error so the orchestrator can
@@ -557,7 +597,7 @@ export async function invokeHook(
         success: false,
         outputs: {},
         error: message,
-        screenshotPath,
+        artifactPaths,
         duration: Date.now() - startTime,
         missingProviderInput: {
           providerModuleId: error.providerModuleId,
@@ -572,8 +612,13 @@ export async function invokeHook(
       success: false,
       outputs: {},
       error: message,
-      screenshotPath,
+      artifactPaths,
       duration: Date.now() - startTime,
     };
+  } finally {
+    // Most hooks write nothing, and every hook invocation would otherwise
+    // leave an empty directory behind. Discard it; a run that produced
+    // artifacts keeps its directory.
+    discardIfEmpty(screenshotDir);
   }
 }

package/src/hooks/test-fixtures/artifact-writing-hook.ts

@@ -0,0 +1,25 @@
+/**
+ * Test fixture: a hook that writes a post-mortem set to its artifact
+ * directory, the way a browser-driven check does — a screenshot, the page
+ * content, and the observed requests.
+ *
+ * Three files with three different extensions on purpose: the executor
+ * used to report only the newest `.png`, which silently discarded the two
+ * that make the screenshot diagnosable.
+ */
+
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { defineHook } from '@celilo/capabilities';
+
+export default defineHook({
+  hook: 'container_created',
+  requires: [],
+  handler: async (ctx) => {
+    const dir = ctx.screenshotDir as string;
+    writeFileSync(join(dir, 'spa-failure.png'), 'not-really-a-png');
+    writeFileSync(join(dir, 'spa-failure.html'), '<html></html>');
+    writeFileSync(join(dir, 'spa-failure.requests.txt'), 'GET /api/getActiveMonth 200');
+    return { api_key: 'wrote-artifacts' };
+  },
+});

package/src/hooks/types.ts

@@ -91,8 +91,13 @@ export interface HookResult {
   success: boolean;
   outputs: Record<string, unknown>;
   error?: string;
-  /** Path to failure screenshot, if captured */
-  screenshotPath?: string;
+  /**
+   * Every file the hook wrote to its per-run artifact directory.
+   * All of them, not just the newest image: a screenshot without the
+   * DOM and the observed requests is the least useful third of a
+   * post-mortem.
+   */
+  artifactPaths?: string[];
   /** Duration in milliseconds */
   duration: number;
   /**
@@ -120,3 +125,16 @@ import type { HookName } from '@celilo/capabilities';
  * Hook manifest section - maps hook names to definitions
  */
 export type HookManifest = Partial<Record<HookName, HookDefinition>>;
+
+/**
+ * Render collected artifacts for an operator-facing error message.
+ *
+ * One definition, because three call sites rendered the single old
+ * `screenshotPath` three separate times and would have drifted the moment
+ * one of them learned about the others.
+ */
+export function describeArtifacts(artifactPaths: string[] | undefined): string {
+  if (!artifactPaths || artifactPaths.length === 0) return '';
+  const label = artifactPaths.length === 1 ? 'Artifact saved' : 'Artifacts saved';
+  return `\n\n${label}:\n  ${artifactPaths.join('\n  ')}`;
+}