disk-space-saviour 0.15.2 → 0.17.0

package/src/scan-state.js

@@ -0,0 +1,239 @@
+/** Advisory discovery/size index and authenticated, short-lived scan plans. */
+import {
+  createHash,
+  createHmac,
+  randomBytes,
+  timingSafeEqual,
+} from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { trace } from './exec.js';
+
+export const REPORT_MAX_AGE = 30 * 60 * 1000;
+export const SIZE_MAX_AGE = 60 * 60 * 1000;
+const LIMIT = 20000;
+
+export function defaultStateDirectory(vars = process.env) {
+  return path.join(
+    vars.XDG_STATE_HOME || path.join(os.homedir(), '.local', 'state'),
+    'disk-space-saviour'
+  );
+}
+
+function payload(report) {
+  const { authentication, ...fields } = report;
+  void authentication;
+  return JSON.stringify(fields);
+}
+
+async function secret(directory) {
+  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+  const file = path.join(directory, 'scan-key');
+  try {
+    await fs.writeFile(file, randomBytes(32), { flag: 'wx', mode: 0o600 });
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
+      throw error;
+    }
+  }
+  return fs.readFile(file);
+}
+
+export async function authenticateReport(report, directory) {
+  if (!directory) {
+    return report;
+  }
+  try {
+    report.authentication = {
+      hostname: os.hostname(),
+      uid: process.getuid?.() ?? os.userInfo().username,
+      digest: createHmac('sha256', await secret(directory))
+        .update(payload(report))
+        .digest('hex'),
+    };
+  } catch (error) {
+    trace('scan state unavailable', error.message);
+  }
+  return report;
+}
+
+function recentLocalReport(report, now) {
+  const age = now - Date.parse(report.createdAt);
+  const auth = report.authentication;
+  if (
+    !Number.isFinite(age) ||
+    age < 0 ||
+    age > REPORT_MAX_AGE ||
+    report.aborted ||
+    auth?.hostname !== os.hostname() ||
+    auth?.uid !== (process.getuid?.() ?? os.userInfo().username)
+  ) {
+    return false;
+  }
+  return true;
+}
+
+export async function authenticatedReport(report, directory, now = Date.now()) {
+  if (
+    !directory ||
+    report?.schema !== 1 ||
+    report.tool !== 'disk-space-saviour' ||
+    !recentLocalReport(report, now)
+  ) {
+    return false;
+  }
+  try {
+    const key = await fs.readFile(path.join(directory, 'scan-key'));
+    const expected = createHmac('sha256', key).update(payload(report)).digest();
+    const actual = Buffer.from(report.authentication.digest, 'hex');
+    return (
+      expected.length === actual.length && timingSafeEqual(expected, actual)
+    );
+  } catch {
+    return false;
+  }
+}
+
+export function scopeKey(options) {
+  const names = [
+    'roots',
+    'scanners',
+    'host',
+    'docker',
+    'dockerDepth',
+    'maxDepth',
+    'staleAgeMs',
+    'inactiveMs',
+    'minSizeBytes',
+    'only',
+    'exclude',
+    'includeVolumes',
+    'noNative',
+    'journalKeepBytes',
+    'containerFilter',
+    'removeStoppedContainers',
+    'investigationMaxAgeMs',
+    'removeContainers',
+    'removeImages',
+    'allowDirtyRepos',
+    'allowDirtyContainers',
+  ];
+  return createHash('sha256')
+    .update(JSON.stringify(names.map((name) => [name, options[name]])))
+    .digest('hex');
+}
+
+export async function saveScan(report, directory) {
+  if (
+    !directory ||
+    !report.authentication ||
+    report.aborted ||
+    report.environments.some((env) => env.scanStatus === 'goal-reached')
+  ) {
+    return;
+  }
+  try {
+    const dir = path.join(directory, 'scans');
+    await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+    const file = path.join(dir, `${scopeKey(report.options)}.json`);
+    const temporary = `${file}.${process.pid}.${randomBytes(4).toString('hex')}`;
+    await fs.writeFile(temporary, JSON.stringify(report), { mode: 0o600 });
+    await fs.rename(temporary, file);
+  } catch (error) {
+    trace('cannot save scan', error.message);
+  }
+}
+
+export async function loadScan(options) {
+  if (!options.cacheDir) {
+    return null;
+  }
+  try {
+    const report = JSON.parse(
+      await fs.readFile(
+        path.join(options.cacheDir, 'scans', `${scopeKey(options)}.json`),
+        'utf8'
+      )
+    );
+    return (await authenticatedReport(report, options.cacheDir, options.now()))
+      ? report
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Cached listings are validated by each directory's own metadata, never its parent alone. */
+export class ScanIndex {
+  constructor(directory) {
+    this.directory = directory;
+    this.entries = new Map();
+    this.dirty = false;
+  }
+
+  async load() {
+    if (!this.directory) {
+      return;
+    }
+    try {
+      const data = JSON.parse(
+        await fs.readFile(
+          path.join(this.directory, 'index', 'directories.json'),
+          'utf8'
+        )
+      );
+      if (data.schema === 1 && Array.isArray(data.entries)) {
+        this.entries = new Map(data.entries.slice(-LIMIT));
+      }
+    } catch {
+      // Missing/corrupt state only loses an optimization.
+    }
+  }
+
+  get(key, fingerprint, maxAge = Infinity) {
+    const entry = this.entries.get(key);
+    return entry &&
+      entry.fingerprint === fingerprint &&
+      Date.now() - entry.time < maxAge
+      ? entry.value
+      : null;
+  }
+
+  set(key, fingerprint, value) {
+    this.entries.delete(key);
+    this.entries.set(key, { fingerprint, value, time: Date.now() });
+    while (this.entries.size > LIMIT) {
+      this.entries.delete(this.entries.keys().next().value);
+    }
+    this.dirty = true;
+  }
+
+  async save() {
+    if (!this.directory || !this.dirty) {
+      return;
+    }
+    const dir = path.join(this.directory, 'index');
+    const temporary = path.join(
+      dir,
+      `directories.${process.pid}.${randomBytes(4).toString('hex')}.json`
+    );
+    try {
+      await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+      await fs.writeFile(
+        temporary,
+        JSON.stringify({ schema: 1, entries: [...this.entries] }),
+        { mode: 0o600 }
+      );
+      await fs.rename(temporary, path.join(dir, 'directories.json'));
+      this.dirty = false;
+    } catch (error) {
+      trace('cannot save discovery index', error.message);
+    } finally {
+      await fs.rm(temporary, { force: true }).catch(() => {});
+    }
+  }
+}
+
+export const directoryFingerprint = (stats) =>
+  `${stats.device}:${stats.inode}:${stats.mtimeMs}:${stats.ctimeMs}`;

package/src/scan.js

@@ -7,9 +7,12 @@ import {
   configureContainerHost,
   resetWritableLayers,
 } from './docker/writable.js';
+import { setInterval, clearInterval } from 'node:timers';
 import { scanDocker } from './docker/scan.js';
 import { LocalEnv } from './env/local.js';
 import { configureScanPolicy } from './env/policy.js';
+import { configureCacheRoots } from './env/cache-roots.js';
+import { checkSignal, environmentScope } from './env/scope.js';
 import { block, tierTotals } from './items.js';
 import { hostExecutor, trace } from './exec.js';
 import { GitInspector } from './git.js';
@@ -21,6 +24,14 @@ import { scanGlobal } from './scanners/global.js';
 import { scanProjects } from './scanners/projects.js';
 import { scanSystem } from './scanners/system.js';
 import { scanVersions } from './scanners/versions.js';
+import { readSwap } from './swap.js';
+import {
+  authenticateReport,
+  authenticatedReport,
+  saveScan,
+  ScanIndex,
+  scopeKey,
+} from './scan-state.js';
 
 export const SCANNERS = {
   projects: scanProjects,
@@ -59,6 +70,9 @@ const RESCAN_OPTIONS = new Set([
   'journalKeep',
   'env',
   'now',
+  'signal',
+  'scanBudget',
+  'onScanProgress',
 ]);
 
 function safetySnapshot(report) {
@@ -70,20 +84,47 @@ function sealReport(report) {
   return report;
 }
 
+function unchangedDiscoveryScope(report, input) {
+  const original = resolveOptions(report.options);
+  const overrides = Object.fromEntries(
+    Object.entries(input).filter(
+      ([name, value]) =>
+        RESCAN_OPTIONS.has(name) && value !== undefined && value !== null
+    )
+  );
+  return (
+    scopeKey(original) ===
+    scopeKey(resolveOptions({ ...original, ...overrides }))
+  );
+}
+
 /** Rebuild scanner-derived actions when a report came from JSON or changed. */
 export async function revalidateReport(saved, input = {}) {
   if (trustedReports.get(saved) === safetySnapshot(saved)) {
     return saved;
   }
-  const scope = {
-    roots: saved.options?.roots,
-    scanners: saved.options?.scanners,
-    containers: saved.options?.containers,
-    host: saved.options?.host,
-    docker: saved.options?.docker,
-    dockerDepth: saved.options?.dockerDepth,
-    maxDepth: saved.options?.maxDepth,
-  };
+  saved = saved.report?.schema === REPORT_SCHEMA ? saved.report : saved;
+  if (
+    unchangedDiscoveryScope(saved, input) &&
+    (await authenticatedReport(
+      saved,
+      input.cacheDir,
+      input.now?.() ?? Date.now()
+    ))
+  ) {
+    return sealReport(saved);
+  }
+  const scope = Object.fromEntries(
+    [
+      'roots',
+      'scanners',
+      'containers',
+      'host',
+      'docker',
+      'dockerDepth',
+      'maxDepth',
+    ].map((name) => [name, saved.options?.[name]])
+  );
   for (const [key, value] of Object.entries(input)) {
     if (RESCAN_OPTIONS.has(key) && value !== undefined && value !== null) {
       scope[key] = value;
@@ -133,6 +174,7 @@ export async function environmentContext(env, options, overrides = {}) {
   });
   await liveness.refresh(true);
   const homes = await env.homeDirs();
+  await configureCacheRoots(env, homes);
   const tmpDirs = await env.tmpDirs();
   const roots = overrides.roots ?? (await defaultRoots(env, homes, tmpDirs));
   return {
@@ -152,16 +194,127 @@ export async function environmentContext(env, options, overrides = {}) {
  * @returns {Promise<object[]>} items
  */
 export async function scanEnvironment(env, options, extra = {}) {
-  const errors = extra.errors ?? [];
-  let context;
+  const controller = new AbortController();
+  const abort = () => controller.abort(options.signal.reason);
+  options.signal?.addEventListener('abort', abort, { once: true });
+  if (options.signal?.aborted) {
+    abort();
+  }
+  const started = Date.now(),
+    deadline = started + options.scanBudgetMs;
+  const timer = setTimeout(
+    () => controller.abort(new Error('scan time budget exceeded')),
+    options.scanBudgetMs
+  );
+  const found = new Map();
+  const completed = new Set();
+  let scanner = 'context';
+  const progress = () =>
+    options.onScanProgress?.({
+      env: env.id,
+      scanner,
+      elapsedMs: Date.now() - started,
+      items: found.size,
+      candidates: found.size,
+      filesSized: env.scanMetrics?.filesSized ?? 0,
+      directoriesSized: env.scanMetrics?.directoriesSized ?? 0,
+      indexHits: env.scanMetrics?.indexHits ?? 0,
+      sizeCacheHits: env.scanMetrics?.sizeCacheHits ?? 0,
+      status: env.scanStatus,
+    });
+  env.scanStatus = 'scanning';
+  const interval = setInterval(progress, 5000);
   try {
-    context = await environmentContext(env, options, extra);
+    return await environmentScope(
+      env,
+      controller.signal,
+      () => {
+        progress();
+        return scanEnvironmentScoped(
+          env,
+          options,
+          extra,
+          found,
+          completed,
+          (name) => {
+            scanner = name;
+            progress();
+          }
+        );
+      },
+      deadline
+    );
   } catch (error) {
-    errors.push({ env: env.id, scanner: 'context', message: error.message });
-    return [];
+    env.scanStatus = options.signal?.aborted
+      ? 'aborted'
+      : controller.signal.aborted
+        ? 'budget-exceeded'
+        : 'error';
+    (extra.errors ?? []).push({ env: env.id, scanner, message: error.message });
+    const items = [...found.values()];
+    for (const item of items) {
+      if (options.signal?.aborted || !completed.has(item.id)) {
+        block(item, `scan incomplete: ${error.message}`);
+      }
+    }
+    return items;
+  } finally {
+    clearTimeout(timer);
+    clearInterval(interval);
+    options.signal?.removeEventListener('abort', abort);
+    progress();
   }
-  const items = [];
+}
+
+async function scanEnvironmentScoped(
+  env,
+  options,
+  extra,
+  found,
+  completed,
+  stage
+) {
+  const errors = extra.errors ?? [];
+  const context = await environmentContext(env, options, extra);
+  context.emit = (item) => found.set(item.id, item);
+  let stopped = false;
+  const ready = async (items) => {
+    await attributeStorage(env, items);
+    for (const item of items) {
+      for (const target of item.paths) {
+        block(item, env.scanPolicy.removalReason(target));
+      }
+      completed.add(item.id);
+    }
+    if (
+      env.kind === 'host' &&
+      context.scanner === 'global' &&
+      options.onScanPhase
+    ) {
+      const result = await options.onScanPhase(
+        sealReport({
+          schema: REPORT_SCHEMA,
+          tool: 'disk-space-saviour',
+          createdAt: new Date(options.now()).toISOString(),
+          host: { env: env.id, label: env.label, platform: env.platform },
+          options: reportOptions(options),
+          environments: [envDescriptor(env, 0, [])],
+          items: filterItems(items, options, [], env.path),
+          totals: tierTotals(items),
+          errors: [],
+          docker: null,
+        })
+      );
+      stopped ||= result === false;
+    }
+    return !stopped;
+  };
+  context.ready = ready;
+  let incomplete = false;
   for (const name of options.scanners) {
+    checkSignal(env.signal);
+    context.scanner = name;
+    stage(name);
     const scanner = SCANNERS[name];
     if (!scanner) {
       throw new Error(
@@ -169,14 +322,31 @@ export async function scanEnvironment(env, options, extra = {}) {
       );
     }
     const started = Date.now();
+    const previous = new Set(found.keys());
     try {
-      items.push(...(await scanner(context)));
+      const entries = await scanner(context);
+      for (const item of entries) {
+        context.emit(item);
+      }
+      await ready(entries.filter((item) => !completed.has(item.id)));
     } catch (error) {
+      checkSignal(env.signal);
+      incomplete = true;
+      for (const [id, item] of found) {
+        if (![previous, completed].some((set) => set.has(id))) {
+          block(item, `scan incomplete: ${error.message}`);
+        }
+      }
       errors.push({ env: env.id, scanner: name, message: error.message });
     }
     trace('scanner', name, 'in', env.id, `${Date.now() - started}ms`);
+    if (stopped) {
+      env.scanStatus = 'goal-reached';
+      return [...found.values()];
+    }
   }
-  await attributeStorage(env, items);
+  stage('storage');
+  const items = [...found.values()];
   for (const item of items) {
     for (const target of item.paths) {
       block(item, env.scanPolicy.removalReason(target));
@@ -185,6 +355,7 @@ export async function scanEnvironment(env, options, extra = {}) {
   for (const message of env.accessErrors?.values() ?? []) {
     errors.push({ env: env.id, scanner: 'filesystem', message });
   }
+  env.scanStatus = incomplete ? 'error' : 'complete';
   return items;
 }
 
@@ -307,13 +478,19 @@ function envDescriptor(env, depth, chain) {
     depth,
     chain,
     hint: env.probeHint ?? null,
+    scanStatus: env.scanStatus ?? null,
+    layerSource: env.writableLayer?.source ?? null,
   };
 }
 
 async function withDisk(env, descriptor) {
   const target = env.platform === 'win32' ? env.currentHome : '/';
   const disk = await env.diskUsage(target).catch(() => null);
-  return { ...descriptor, disk: disk ? { path: target, ...disk } : null };
+  return {
+    ...descriptor,
+    disk: disk ? { path: target, ...disk } : null,
+    swap: await readSwap(env),
+  };
 }
 
 async function dockerSection(env, options, errors) {
@@ -373,10 +550,43 @@ function blockMountedHostItems(items, mounts, pathApi) {
 }
 
 export async function scan(input = {}) {
+  const env = input.env ?? new LocalEnv();
+  try {
+    return await environmentScope(env, input.signal, () =>
+      scanScoped({ ...input, env })
+    );
+  } catch (error) {
+    if (!input.signal?.aborted) {
+      throw error;
+    }
+    env.scanStatus = 'aborted';
+    const options = resolveOptions(input);
+    return buildReport({
+      options,
+      env,
+      items: [],
+      environments: [envDescriptor(env, 0, [])],
+      docker: null,
+      errors: [{ env: env.id, scanner: 'context', message: error.message }],
+      startedAt: options.now(),
+    });
+  }
+}
+
+async function scanScoped(input) {
   const options = resolveOptions(input);
   const env = input.env ?? new LocalEnv();
   const startedAt = options.now();
   const errors = [];
+  env.scanMetrics = {
+    filesSized: 0,
+    directoriesSized: 0,
+    indexHits: 0,
+    sizeCacheHits: 0,
+  };
+  env.scanIndex = new ScanIndex(options.cacheDir);
+  env.platformIndex = options.platformIndex;
+  await env.scanIndex.load();
   resetWritableLayers(env.executor);
   env.writableLayer = null;
   env.scanPolicy = null;
@@ -390,7 +600,10 @@ export async function scan(input = {}) {
     options.host === false
       ? []
       : await scanEnvironment(env, options, { roots: options.roots, errors });
-  const docker = await dockerSection(env, options, errors);
+  const docker =
+    env.scanStatus === 'goal-reached'
+      ? null
+      : await dockerSection(env, options, errors);
   blockMountedHostItems(hostItems, docker?.bindMounts ?? [], env.path);
   const items = filterItems(
     [...hostItems, ...(docker?.items ?? [])],
@@ -413,6 +626,10 @@ export async function scan(input = {}) {
     startedAt,
   });
   reportEnvironments.set(report, env);
+  await env.scanIndex.save();
+  await authenticateReport(report, options.cacheDir);
+  sealReport(report);
+  await saveScan(report, options.cacheDir);
   return report;
 }
 
@@ -420,6 +637,9 @@ function reportOptions(options) {
   const serializable = { ...options };
   delete serializable.now;
   delete serializable.env;
+  delete serializable.signal;
+  delete serializable.onScanProgress;
+  delete serializable.onScanPhase;
   return serializable;
 }
 
@@ -440,6 +660,7 @@ export function buildReport({
     tool: 'disk-space-saviour',
     createdAt: new Date(options.now()).toISOString(),
     durationMs: options.now() - startedAt,
+    aborted: Boolean(options.signal?.aborted),
     host: { env: env.id, label: env.label, platform: env.platform },
     options: reportOptions(options),
     environments,

package/src/scanners/agents.js

@@ -1,3 +1,4 @@
+import { scanItems } from '../env/scope.js';
 /**
  * Snapshot stores of AI coding agents built on the opencode storage layout
  * (`@link-assistant/agent`, opencode).
@@ -112,7 +113,7 @@ async function appItems(context, app, root) {
       mtimeMs: usages.get(state.entry.path)?.newestMtimeMs ?? 0,
     }))
     .sort((a, b) => b.mtimeMs - a.mtimeMs)[0].state;
-  const items = [];
+  const items = scanItems(context);
   for (const state of states) {
     const usage = usages.get(state.entry.path);
     const verdict =
@@ -148,7 +149,7 @@ async function appItems(context, app, root) {
 export async function scanAgents(context, { apps = AGENT_APPS } = {}) {
   const { env, options } = context;
   const homes = options.homes ?? (await env.homeDirs());
-  const items = [];
+  const items = scanItems(context);
   const seen = new Set();
   for (const home of homes) {
     for (const dataDir of dataDirs(env, home)) {

package/src/scanners/common.js

@@ -26,10 +26,15 @@ export async function listMany(env, dirs) {
  * @returns {string|null}
  */
 export async function scanTimeBusy(context, item) {
-  await context.liveness?.resolve(item.paths);
+  await resolvePaths(context, item.paths);
   return context.liveness ? context.liveness.busyReason(item) : null;
 }
 
+/** Prime one batch before constructing items, so later per-item probes are cached. */
+export async function resolvePaths(context, paths) {
+  await context.liveness?.resolve(paths);
+}
+
 /**
  * Whether `now - timestamp` exceeds `windowMs` (unknown timestamps count as
  * recent, so they are never treated as abandoned).