disk-space-saviour 0.16.0 → 0.17.0

package/src/options.js

@@ -16,7 +16,7 @@ export const DEFAULTS = {
   docker: null,
   dockerDepth: 3,
   containers: [],
-  scanners: ['projects', 'global', 'versions', 'agents', 'system'],
+  scanners: ['global', 'projects', 'versions', 'agents', 'system'],
   only: [],
   exclude: [],
   includeVolumes: false,
@@ -33,6 +33,8 @@ export const DEFAULTS = {
   backupDir: null,
   auditDir: null,
   scanBudget: '2m',
+  cacheDir: false,
+  platformIndex: true,
 };
 
 function toBytes(value) {

package/src/report.js

@@ -56,7 +56,7 @@ function itemLine(item, showEnv) {
     : item.imageBytes
       ? `  (${formatBytes(item.imageBytes)} in image${item.imageRef ? ` ${item.imageRef}` : ''}; 0 B of that data freeable now; remove the stopped container and unused image after the task completes)`
       : '';
-  return `  ${pad(formatBytes(item.bytes), 10)}  ${env}${item.rule}  ${itemLocation(item)}${confirm}${storage}`;
+  return `  ${pad(formatBytes(item.bytes), 10)}${item.sizeEstimated ? ' (estimated)' : ''}  ${env}${item.rule}  ${itemLocation(item)}${confirm}${storage}`;
 }
 
 function limited(lines, limit) {
@@ -318,6 +318,18 @@ export function formatReport(report, options = {}) {
     `disk-space-saviour scan of ${host.label} in ${(report.durationMs / 1000).toFixed(1)}s (report only, nothing was deleted)`,
     formatDisk(host.disk),
   ];
+  if (host.swap) {
+    lines.push(
+      `Swap: ${formatBytes(host.swap.used)} used of ${formatBytes(host.swap.total)} (managed by macOS)`
+    );
+    lines.push(host.swap.attribution);
+    for (const holder of host.swap.holders) {
+      lines.push(
+        `  ${holder.pid} ${holder.command}: ${formatBytes(holder.memoryBytes)} memory, ${formatBytes(holder.compressedBytes)} compressed`
+      );
+    }
+    lines.push(host.swap.advice);
+  }
   for (const tier of TIERS) {
     lines.push(...tierSection(report, tier, { limit }));
   }
@@ -325,7 +337,11 @@ export function formatReport(report, options = {}) {
     if (env.layerSource) {
       lines.push(`  ${env.label}: layer source ${env.layerSource}`);
     }
-    if (env.scanStatus && env.scanStatus !== 'complete') {
+    if (env.scanStatus === 'goal-reached') {
+      lines.push(
+        `  ${env.label}: scan stopped after the free-space goal was met`
+      );
+    } else if (env.scanStatus && env.scanStatus !== 'complete') {
       lines.push(`  ${env.label}: scan incomplete (${env.scanStatus})`);
     }
     if (env.hint) {

package/src/scan-state.js

@@ -0,0 +1,239 @@
+/** Advisory discovery/size index and authenticated, short-lived scan plans. */
+import {
+  createHash,
+  createHmac,
+  randomBytes,
+  timingSafeEqual,
+} from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { trace } from './exec.js';
+
+export const REPORT_MAX_AGE = 30 * 60 * 1000;
+export const SIZE_MAX_AGE = 60 * 60 * 1000;
+const LIMIT = 20000;
+
+export function defaultStateDirectory(vars = process.env) {
+  return path.join(
+    vars.XDG_STATE_HOME || path.join(os.homedir(), '.local', 'state'),
+    'disk-space-saviour'
+  );
+}
+
+function payload(report) {
+  const { authentication, ...fields } = report;
+  void authentication;
+  return JSON.stringify(fields);
+}
+
+async function secret(directory) {
+  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+  const file = path.join(directory, 'scan-key');
+  try {
+    await fs.writeFile(file, randomBytes(32), { flag: 'wx', mode: 0o600 });
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
+      throw error;
+    }
+  }
+  return fs.readFile(file);
+}
+
+export async function authenticateReport(report, directory) {
+  if (!directory) {
+    return report;
+  }
+  try {
+    report.authentication = {
+      hostname: os.hostname(),
+      uid: process.getuid?.() ?? os.userInfo().username,
+      digest: createHmac('sha256', await secret(directory))
+        .update(payload(report))
+        .digest('hex'),
+    };
+  } catch (error) {
+    trace('scan state unavailable', error.message);
+  }
+  return report;
+}
+
+function recentLocalReport(report, now) {
+  const age = now - Date.parse(report.createdAt);
+  const auth = report.authentication;
+  if (
+    !Number.isFinite(age) ||
+    age < 0 ||
+    age > REPORT_MAX_AGE ||
+    report.aborted ||
+    auth?.hostname !== os.hostname() ||
+    auth?.uid !== (process.getuid?.() ?? os.userInfo().username)
+  ) {
+    return false;
+  }
+  return true;
+}
+
+export async function authenticatedReport(report, directory, now = Date.now()) {
+  if (
+    !directory ||
+    report?.schema !== 1 ||
+    report.tool !== 'disk-space-saviour' ||
+    !recentLocalReport(report, now)
+  ) {
+    return false;
+  }
+  try {
+    const key = await fs.readFile(path.join(directory, 'scan-key'));
+    const expected = createHmac('sha256', key).update(payload(report)).digest();
+    const actual = Buffer.from(report.authentication.digest, 'hex');
+    return (
+      expected.length === actual.length && timingSafeEqual(expected, actual)
+    );
+  } catch {
+    return false;
+  }
+}
+
+export function scopeKey(options) {
+  const names = [
+    'roots',
+    'scanners',
+    'host',
+    'docker',
+    'dockerDepth',
+    'maxDepth',
+    'staleAgeMs',
+    'inactiveMs',
+    'minSizeBytes',
+    'only',
+    'exclude',
+    'includeVolumes',
+    'noNative',
+    'journalKeepBytes',
+    'containerFilter',
+    'removeStoppedContainers',
+    'investigationMaxAgeMs',
+    'removeContainers',
+    'removeImages',
+    'allowDirtyRepos',
+    'allowDirtyContainers',
+  ];
+  return createHash('sha256')
+    .update(JSON.stringify(names.map((name) => [name, options[name]])))
+    .digest('hex');
+}
+
+export async function saveScan(report, directory) {
+  if (
+    !directory ||
+    !report.authentication ||
+    report.aborted ||
+    report.environments.some((env) => env.scanStatus === 'goal-reached')
+  ) {
+    return;
+  }
+  try {
+    const dir = path.join(directory, 'scans');
+    await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+    const file = path.join(dir, `${scopeKey(report.options)}.json`);
+    const temporary = `${file}.${process.pid}.${randomBytes(4).toString('hex')}`;
+    await fs.writeFile(temporary, JSON.stringify(report), { mode: 0o600 });
+    await fs.rename(temporary, file);
+  } catch (error) {
+    trace('cannot save scan', error.message);
+  }
+}
+
+export async function loadScan(options) {
+  if (!options.cacheDir) {
+    return null;
+  }
+  try {
+    const report = JSON.parse(
+      await fs.readFile(
+        path.join(options.cacheDir, 'scans', `${scopeKey(options)}.json`),
+        'utf8'
+      )
+    );
+    return (await authenticatedReport(report, options.cacheDir, options.now()))
+      ? report
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Cached listings are validated by each directory's own metadata, never its parent alone. */
+export class ScanIndex {
+  constructor(directory) {
+    this.directory = directory;
+    this.entries = new Map();
+    this.dirty = false;
+  }
+
+  async load() {
+    if (!this.directory) {
+      return;
+    }
+    try {
+      const data = JSON.parse(
+        await fs.readFile(
+          path.join(this.directory, 'index', 'directories.json'),
+          'utf8'
+        )
+      );
+      if (data.schema === 1 && Array.isArray(data.entries)) {
+        this.entries = new Map(data.entries.slice(-LIMIT));
+      }
+    } catch {
+      // Missing/corrupt state only loses an optimization.
+    }
+  }
+
+  get(key, fingerprint, maxAge = Infinity) {
+    const entry = this.entries.get(key);
+    return entry &&
+      entry.fingerprint === fingerprint &&
+      Date.now() - entry.time < maxAge
+      ? entry.value
+      : null;
+  }
+
+  set(key, fingerprint, value) {
+    this.entries.delete(key);
+    this.entries.set(key, { fingerprint, value, time: Date.now() });
+    while (this.entries.size > LIMIT) {
+      this.entries.delete(this.entries.keys().next().value);
+    }
+    this.dirty = true;
+  }
+
+  async save() {
+    if (!this.directory || !this.dirty) {
+      return;
+    }
+    const dir = path.join(this.directory, 'index');
+    const temporary = path.join(
+      dir,
+      `directories.${process.pid}.${randomBytes(4).toString('hex')}.json`
+    );
+    try {
+      await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+      await fs.writeFile(
+        temporary,
+        JSON.stringify({ schema: 1, entries: [...this.entries] }),
+        { mode: 0o600 }
+      );
+      await fs.rename(temporary, path.join(dir, 'directories.json'));
+      this.dirty = false;
+    } catch (error) {
+      trace('cannot save discovery index', error.message);
+    } finally {
+      await fs.rm(temporary, { force: true }).catch(() => {});
+    }
+  }
+}
+
+export const directoryFingerprint = (stats) =>
+  `${stats.device}:${stats.inode}:${stats.mtimeMs}:${stats.ctimeMs}`;

package/src/scan.js

@@ -24,6 +24,14 @@ import { scanGlobal } from './scanners/global.js';
 import { scanProjects } from './scanners/projects.js';
 import { scanSystem } from './scanners/system.js';
 import { scanVersions } from './scanners/versions.js';
+import { readSwap } from './swap.js';
+import {
+  authenticateReport,
+  authenticatedReport,
+  saveScan,
+  ScanIndex,
+  scopeKey,
+} from './scan-state.js';
 
 export const SCANNERS = {
   projects: scanProjects,
@@ -76,20 +84,47 @@ function sealReport(report) {
   return report;
 }
 
+function unchangedDiscoveryScope(report, input) {
+  const original = resolveOptions(report.options);
+  const overrides = Object.fromEntries(
+    Object.entries(input).filter(
+      ([name, value]) =>
+        RESCAN_OPTIONS.has(name) && value !== undefined && value !== null
+    )
+  );
+  return (
+    scopeKey(original) ===
+    scopeKey(resolveOptions({ ...original, ...overrides }))
+  );
+}
+
 /** Rebuild scanner-derived actions when a report came from JSON or changed. */
 export async function revalidateReport(saved, input = {}) {
   if (trustedReports.get(saved) === safetySnapshot(saved)) {
     return saved;
   }
-  const scope = {
-    roots: saved.options?.roots,
-    scanners: saved.options?.scanners,
-    containers: saved.options?.containers,
-    host: saved.options?.host,
-    docker: saved.options?.docker,
-    dockerDepth: saved.options?.dockerDepth,
-    maxDepth: saved.options?.maxDepth,
-  };
+  saved = saved.report?.schema === REPORT_SCHEMA ? saved.report : saved;
+  if (
+    unchangedDiscoveryScope(saved, input) &&
+    (await authenticatedReport(
+      saved,
+      input.cacheDir,
+      input.now?.() ?? Date.now()
+    ))
+  ) {
+    return sealReport(saved);
+  }
+  const scope = Object.fromEntries(
+    [
+      'roots',
+      'scanners',
+      'containers',
+      'host',
+      'docker',
+      'dockerDepth',
+      'maxDepth',
+    ].map((name) => [name, saved.options?.[name]])
+  );
   for (const [key, value] of Object.entries(input)) {
     if (RESCAN_OPTIONS.has(key) && value !== undefined && value !== null) {
       scope[key] = value;
@@ -172,6 +207,7 @@ export async function scanEnvironment(env, options, extra = {}) {
     options.scanBudgetMs
   );
   const found = new Map();
+  const completed = new Set();
   let scanner = 'context';
   const progress = () =>
     options.onScanProgress?.({
@@ -179,6 +215,11 @@ export async function scanEnvironment(env, options, extra = {}) {
       scanner,
       elapsedMs: Date.now() - started,
       items: found.size,
+      candidates: found.size,
+      filesSized: env.scanMetrics?.filesSized ?? 0,
+      directoriesSized: env.scanMetrics?.directoriesSized ?? 0,
+      indexHits: env.scanMetrics?.indexHits ?? 0,
+      sizeCacheHits: env.scanMetrics?.sizeCacheHits ?? 0,
       status: env.scanStatus,
     });
   env.scanStatus = 'scanning';
@@ -189,10 +230,17 @@ export async function scanEnvironment(env, options, extra = {}) {
       controller.signal,
       () => {
         progress();
-        return scanEnvironmentScoped(env, options, extra, found, (name) => {
-          scanner = name;
-          progress();
-        });
+        return scanEnvironmentScoped(
+          env,
+          options,
+          extra,
+          found,
+          completed,
+          (name) => {
+            scanner = name;
+            progress();
+          }
+        );
       },
       deadline
     );
@@ -205,7 +253,9 @@ export async function scanEnvironment(env, options, extra = {}) {
     (extra.errors ?? []).push({ env: env.id, scanner, message: error.message });
     const items = [...found.values()];
     for (const item of items) {
-      block(item, `scan incomplete: ${error.message}`);
+      if (options.signal?.aborted || !completed.has(item.id)) {
+        block(item, `scan incomplete: ${error.message}`);
+      }
     }
     return items;
   } finally {
@@ -216,13 +266,54 @@ export async function scanEnvironment(env, options, extra = {}) {
   }
 }
 
-async function scanEnvironmentScoped(env, options, extra, found, stage) {
+async function scanEnvironmentScoped(
+  env,
+  options,
+  extra,
+  found,
+  completed,
+  stage
+) {
   const errors = extra.errors ?? [];
   const context = await environmentContext(env, options, extra);
   context.emit = (item) => found.set(item.id, item);
+  let stopped = false;
+  const ready = async (items) => {
+    await attributeStorage(env, items);
+    for (const item of items) {
+      for (const target of item.paths) {
+        block(item, env.scanPolicy.removalReason(target));
+      }
+      completed.add(item.id);
+    }
+    if (
+      env.kind === 'host' &&
+      context.scanner === 'global' &&
+      options.onScanPhase
+    ) {
+      const result = await options.onScanPhase(
+        sealReport({
+          schema: REPORT_SCHEMA,
+          tool: 'disk-space-saviour',
+          createdAt: new Date(options.now()).toISOString(),
+          host: { env: env.id, label: env.label, platform: env.platform },
+          options: reportOptions(options),
+          environments: [envDescriptor(env, 0, [])],
+          items: filterItems(items, options, [], env.path),
+          totals: tierTotals(items),
+          errors: [],
+          docker: null,
+        })
+      );
+      stopped ||= result === false;
+    }
+    return !stopped;
+  };
+  context.ready = ready;
   let incomplete = false;
   for (const name of options.scanners) {
     checkSignal(env.signal);
+    context.scanner = name;
     stage(name);
     const scanner = SCANNERS[name];
     if (!scanner) {
@@ -237,21 +328,25 @@ async function scanEnvironmentScoped(env, options, extra, found, stage) {
       for (const item of entries) {
         context.emit(item);
       }
+      await ready(entries.filter((item) => !completed.has(item.id)));
     } catch (error) {
       checkSignal(env.signal);
       incomplete = true;
       for (const [id, item] of found) {
-        if (!previous.has(id)) {
+        if (![previous, completed].some((set) => set.has(id))) {
           block(item, `scan incomplete: ${error.message}`);
         }
       }
       errors.push({ env: env.id, scanner: name, message: error.message });
     }
     trace('scanner', name, 'in', env.id, `${Date.now() - started}ms`);
+    if (stopped) {
+      env.scanStatus = 'goal-reached';
+      return [...found.values()];
+    }
   }
   stage('storage');
   const items = [...found.values()];
-  await attributeStorage(env, items);
   for (const item of items) {
     for (const target of item.paths) {
       block(item, env.scanPolicy.removalReason(target));
@@ -391,7 +486,11 @@ function envDescriptor(env, depth, chain) {
 async function withDisk(env, descriptor) {
   const target = env.platform === 'win32' ? env.currentHome : '/';
   const disk = await env.diskUsage(target).catch(() => null);
-  return { ...descriptor, disk: disk ? { path: target, ...disk } : null };
+  return {
+    ...descriptor,
+    disk: disk ? { path: target, ...disk } : null,
+    swap: await readSwap(env),
+  };
 }
 
 async function dockerSection(env, options, errors) {
@@ -479,6 +578,15 @@ async function scanScoped(input) {
   const env = input.env ?? new LocalEnv();
   const startedAt = options.now();
   const errors = [];
+  env.scanMetrics = {
+    filesSized: 0,
+    directoriesSized: 0,
+    indexHits: 0,
+    sizeCacheHits: 0,
+  };
+  env.scanIndex = new ScanIndex(options.cacheDir);
+  env.platformIndex = options.platformIndex;
+  await env.scanIndex.load();
   resetWritableLayers(env.executor);
   env.writableLayer = null;
   env.scanPolicy = null;
@@ -492,7 +600,10 @@ async function scanScoped(input) {
     options.host === false
       ? []
       : await scanEnvironment(env, options, { roots: options.roots, errors });
-  const docker = await dockerSection(env, options, errors);
+  const docker =
+    env.scanStatus === 'goal-reached'
+      ? null
+      : await dockerSection(env, options, errors);
   blockMountedHostItems(hostItems, docker?.bindMounts ?? [], env.path);
   const items = filterItems(
     [...hostItems, ...(docker?.items ?? [])],
@@ -515,6 +626,10 @@ async function scanScoped(input) {
     startedAt,
   });
   reportEnvironments.set(report, env);
+  await env.scanIndex.save();
+  await authenticateReport(report, options.cacheDir);
+  sealReport(report);
+  await saveScan(report, options.cacheDir);
   return report;
 }
 
@@ -524,6 +639,7 @@ function reportOptions(options) {
   delete serializable.env;
   delete serializable.signal;
   delete serializable.onScanProgress;
+  delete serializable.onScanPhase;
   return serializable;
 }
 

package/src/scanners/global.js

@@ -135,6 +135,10 @@ function ageFilter(context, rule) {
   return () => true;
 }
 
+function requiresMtime(rule) {
+  return rule.mtime ?? (rule.scope === 'tmp' || Boolean(rule.minAge));
+}
+
 /** Keep the latest installed revision of each browser and platform family. */
 function olderVersions(env, rule, paths, stats) {
   if (!rule.versionPattern) {
@@ -181,6 +185,7 @@ async function buildItem(context, rule, base, group) {
     path: paths[0],
     paths,
     bytes: group.reduce((sum, entry) => sum + entry.usage.bytes, 0),
+    sizeEstimated: group.some((entry) => entry.usage.estimated),
     newestMtimeMs: Math.max(...group.map((e) => e.usage.newestMtimeMs ?? 0)),
     tier: rule.tier ?? 'safe',
     reason: rule.minAge
@@ -269,13 +274,21 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       found.push({ rule, base, paths });
     }
   }
-  const usages = await env.usageMany(found.flatMap((entry) => entry.paths));
   await resolvePaths(
     context,
     found.flatMap((entry) => entry.paths)
   );
   const items = scanItems(context);
+  // Deliver cheap native measurements before rules needing recursive activity
+  // timestamps, so an emergency goal can avoid the expensive work entirely.
+  found.sort(
+    (a, b) => Number(requiresMtime(a.rule)) - Number(requiresMtime(b.rule))
+  );
   for (const { rule, base, paths } of found) {
+    const usages = await env.usageMany(paths, {
+      mtime: requiresMtime(rule),
+      cache: true,
+    });
     const accept = ageFilter(context, rule);
     const measured = paths
       .map((target) => ({ path: target, usage: usages.get(target) }))
@@ -284,7 +297,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       ? [measured]
       : measured.map((entry) => [entry]);
     for (const group of groups.filter((g) => g.length > 0)) {
-      items.push(await buildItem(context, rule, base, group));
+      const item = await buildItem(context, rule, base, group);
+      items.push(item);
+      if ((await context.ready?.([item])) === false) {
+        return items;
+      }
     }
   }
   return items;