disk-space-saviour 0.16.0 → 0.18.0

package/src/scan-state.js

@@ -0,0 +1,239 @@
+/** Advisory discovery/size index and authenticated, short-lived scan plans. */
+import {
+  createHash,
+  createHmac,
+  randomBytes,
+  timingSafeEqual,
+} from 'node:crypto';
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { trace } from './exec.js';
+
+export const REPORT_MAX_AGE = 30 * 60 * 1000;
+export const SIZE_MAX_AGE = 60 * 60 * 1000;
+const LIMIT = 20000;
+
+export function defaultStateDirectory(vars = process.env) {
+  return path.join(
+    vars.XDG_STATE_HOME || path.join(os.homedir(), '.local', 'state'),
+    'disk-space-saviour'
+  );
+}
+
+function payload(report) {
+  const { authentication, ...fields } = report;
+  void authentication;
+  return JSON.stringify(fields);
+}
+
+async function secret(directory) {
+  await fs.mkdir(directory, { recursive: true, mode: 0o700 });
+  const file = path.join(directory, 'scan-key');
+  try {
+    await fs.writeFile(file, randomBytes(32), { flag: 'wx', mode: 0o600 });
+  } catch (error) {
+    if (error.code !== 'EEXIST') {
+      throw error;
+    }
+  }
+  return fs.readFile(file);
+}
+
+export async function authenticateReport(report, directory) {
+  if (!directory) {
+    return report;
+  }
+  try {
+    report.authentication = {
+      hostname: os.hostname(),
+      uid: process.getuid?.() ?? os.userInfo().username,
+      digest: createHmac('sha256', await secret(directory))
+        .update(payload(report))
+        .digest('hex'),
+    };
+  } catch (error) {
+    trace('scan state unavailable', error.message);
+  }
+  return report;
+}
+
+function recentLocalReport(report, now) {
+  const age = now - Date.parse(report.createdAt);
+  const auth = report.authentication;
+  if (
+    !Number.isFinite(age) ||
+    age < 0 ||
+    age > REPORT_MAX_AGE ||
+    report.aborted ||
+    auth?.hostname !== os.hostname() ||
+    auth?.uid !== (process.getuid?.() ?? os.userInfo().username)
+  ) {
+    return false;
+  }
+  return true;
+}
+
+export async function authenticatedReport(report, directory, now = Date.now()) {
+  if (
+    !directory ||
+    report?.schema !== 1 ||
+    report.tool !== 'disk-space-saviour' ||
+    !recentLocalReport(report, now)
+  ) {
+    return false;
+  }
+  try {
+    const key = await fs.readFile(path.join(directory, 'scan-key'));
+    const expected = createHmac('sha256', key).update(payload(report)).digest();
+    const actual = Buffer.from(report.authentication.digest, 'hex');
+    return (
+      expected.length === actual.length && timingSafeEqual(expected, actual)
+    );
+  } catch {
+    return false;
+  }
+}
+
+export function scopeKey(options) {
+  const names = [
+    'roots',
+    'scanners',
+    'host',
+    'docker',
+    'dockerDepth',
+    'maxDepth',
+    'staleAgeMs',
+    'inactiveMs',
+    'minSizeBytes',
+    'only',
+    'exclude',
+    'includeVolumes',
+    'noNative',
+    'journalKeepBytes',
+    'containerFilter',
+    'removeStoppedContainers',
+    'investigationMaxAgeMs',
+    'removeContainers',
+    'removeImages',
+    'allowDirtyRepos',
+    'allowDirtyContainers',
+  ];
+  return createHash('sha256')
+    .update(JSON.stringify(names.map((name) => [name, options[name]])))
+    .digest('hex');
+}
+
+export async function saveScan(report, directory) {
+  if (
+    !directory ||
+    !report.authentication ||
+    report.aborted ||
+    report.environments.some((env) => env.scanStatus === 'goal-reached')
+  ) {
+    return;
+  }
+  try {
+    const dir = path.join(directory, 'scans');
+    await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+    const file = path.join(dir, `${scopeKey(report.options)}.json`);
+    const temporary = `${file}.${process.pid}.${randomBytes(4).toString('hex')}`;
+    await fs.writeFile(temporary, JSON.stringify(report), { mode: 0o600 });
+    await fs.rename(temporary, file);
+  } catch (error) {
+    trace('cannot save scan', error.message);
+  }
+}
+
+export async function loadScan(options) {
+  if (!options.cacheDir) {
+    return null;
+  }
+  try {
+    const report = JSON.parse(
+      await fs.readFile(
+        path.join(options.cacheDir, 'scans', `${scopeKey(options)}.json`),
+        'utf8'
+      )
+    );
+    return (await authenticatedReport(report, options.cacheDir, options.now()))
+      ? report
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Cached listings are validated by each directory's own metadata, never its parent alone. */
+export class ScanIndex {
+  constructor(directory) {
+    this.directory = directory;
+    this.entries = new Map();
+    this.dirty = false;
+  }
+
+  async load() {
+    if (!this.directory) {
+      return;
+    }
+    try {
+      const data = JSON.parse(
+        await fs.readFile(
+          path.join(this.directory, 'index', 'directories.json'),
+          'utf8'
+        )
+      );
+      if (data.schema === 1 && Array.isArray(data.entries)) {
+        this.entries = new Map(data.entries.slice(-LIMIT));
+      }
+    } catch {
+      // Missing/corrupt state only loses an optimization.
+    }
+  }
+
+  get(key, fingerprint, maxAge = Infinity) {
+    const entry = this.entries.get(key);
+    return entry &&
+      entry.fingerprint === fingerprint &&
+      Date.now() - entry.time < maxAge
+      ? entry.value
+      : null;
+  }
+
+  set(key, fingerprint, value) {
+    this.entries.delete(key);
+    this.entries.set(key, { fingerprint, value, time: Date.now() });
+    while (this.entries.size > LIMIT) {
+      this.entries.delete(this.entries.keys().next().value);
+    }
+    this.dirty = true;
+  }
+
+  async save() {
+    if (!this.directory || !this.dirty) {
+      return;
+    }
+    const dir = path.join(this.directory, 'index');
+    const temporary = path.join(
+      dir,
+      `directories.${process.pid}.${randomBytes(4).toString('hex')}.json`
+    );
+    try {
+      await fs.mkdir(dir, { recursive: true, mode: 0o700 });
+      await fs.writeFile(
+        temporary,
+        JSON.stringify({ schema: 1, entries: [...this.entries] }),
+        { mode: 0o600 }
+      );
+      await fs.rename(temporary, path.join(dir, 'directories.json'));
+      this.dirty = false;
+    } catch (error) {
+      trace('cannot save discovery index', error.message);
+    } finally {
+      await fs.rm(temporary, { force: true }).catch(() => {});
+    }
+  }
+}
+
+export const directoryFingerprint = (stats) =>
+  `${stats.device}:${stats.inode}:${stats.mtimeMs}:${stats.ctimeMs}`;

package/src/scan.js

@@ -24,6 +24,15 @@ import { scanGlobal } from './scanners/global.js';
 import { scanProjects } from './scanners/projects.js';
 import { scanSystem } from './scanners/system.js';
 import { scanVersions } from './scanners/versions.js';
+import { readSwap } from './swap.js';
+import {
+  authenticateReport,
+  authenticatedReport,
+  saveScan,
+  ScanIndex,
+  scopeKey,
+} from './scan-state.js';
+import { PhaseTimer, timed } from './timings.js';
 
 export const SCANNERS = {
   projects: scanProjects,
@@ -76,20 +85,47 @@ function sealReport(report) {
   return report;
 }
 
+function unchangedDiscoveryScope(report, input) {
+  const original = resolveOptions(report.options);
+  const overrides = Object.fromEntries(
+    Object.entries(input).filter(
+      ([name, value]) =>
+        RESCAN_OPTIONS.has(name) && value !== undefined && value !== null
+    )
+  );
+  return (
+    scopeKey(original) ===
+    scopeKey(resolveOptions({ ...original, ...overrides }))
+  );
+}
+
 /** Rebuild scanner-derived actions when a report came from JSON or changed. */
 export async function revalidateReport(saved, input = {}) {
   if (trustedReports.get(saved) === safetySnapshot(saved)) {
     return saved;
   }
-  const scope = {
-    roots: saved.options?.roots,
-    scanners: saved.options?.scanners,
-    containers: saved.options?.containers,
-    host: saved.options?.host,
-    docker: saved.options?.docker,
-    dockerDepth: saved.options?.dockerDepth,
-    maxDepth: saved.options?.maxDepth,
-  };
+  saved = saved.report?.schema === REPORT_SCHEMA ? saved.report : saved;
+  if (
+    unchangedDiscoveryScope(saved, input) &&
+    (await authenticatedReport(
+      saved,
+      input.cacheDir,
+      input.now?.() ?? Date.now()
+    ))
+  ) {
+    return sealReport(saved);
+  }
+  const scope = Object.fromEntries(
+    [
+      'roots',
+      'scanners',
+      'containers',
+      'host',
+      'docker',
+      'dockerDepth',
+      'maxDepth',
+    ].map((name) => [name, saved.options?.[name]])
+  );
   for (const [key, value] of Object.entries(input)) {
     if (RESCAN_OPTIONS.has(key) && value !== undefined && value !== null) {
       scope[key] = value;
@@ -172,6 +208,7 @@ export async function scanEnvironment(env, options, extra = {}) {
     options.scanBudgetMs
   );
   const found = new Map();
+  const completed = new Set();
   let scanner = 'context';
   const progress = () =>
     options.onScanProgress?.({
@@ -179,6 +216,11 @@ export async function scanEnvironment(env, options, extra = {}) {
       scanner,
       elapsedMs: Date.now() - started,
       items: found.size,
+      candidates: found.size,
+      filesSized: env.scanMetrics?.filesSized ?? 0,
+      directoriesSized: env.scanMetrics?.directoriesSized ?? 0,
+      indexHits: env.scanMetrics?.indexHits ?? 0,
+      sizeCacheHits: env.scanMetrics?.sizeCacheHits ?? 0,
       status: env.scanStatus,
     });
   env.scanStatus = 'scanning';
@@ -189,10 +231,17 @@ export async function scanEnvironment(env, options, extra = {}) {
       controller.signal,
       () => {
         progress();
-        return scanEnvironmentScoped(env, options, extra, found, (name) => {
-          scanner = name;
-          progress();
-        });
+        return scanEnvironmentScoped(
+          env,
+          options,
+          extra,
+          found,
+          completed,
+          (name) => {
+            scanner = name;
+            progress();
+          }
+        );
       },
       deadline
     );
@@ -205,7 +254,9 @@ export async function scanEnvironment(env, options, extra = {}) {
     (extra.errors ?? []).push({ env: env.id, scanner, message: error.message });
     const items = [...found.values()];
     for (const item of items) {
-      block(item, `scan incomplete: ${error.message}`);
+      if (options.signal?.aborted || !completed.has(item.id)) {
+        block(item, `scan incomplete: ${error.message}`);
+      }
     }
     return items;
   } finally {
@@ -216,13 +267,63 @@ export async function scanEnvironment(env, options, extra = {}) {
   }
 }
 
-async function scanEnvironmentScoped(env, options, extra, found, stage) {
+/** Phase timings of an environment, started on its first scan. */
+function environmentTimings(env) {
+  env.timings ??= new PhaseTimer();
+  return env.timings;
+}
+
+async function scanEnvironmentScoped(
+  env,
+  options,
+  extra,
+  found,
+  completed,
+  stage
+) {
   const errors = extra.errors ?? [];
-  const context = await environmentContext(env, options, extra);
+  const context = await environmentTimings(env).measure('context', () =>
+    environmentContext(env, options, extra)
+  );
+  context.timings = env.timings;
   context.emit = (item) => found.set(item.id, item);
+  let stopped = false;
+  const ready = async (items) => {
+    await env.timings.measure('storage', () => attributeStorage(env, items));
+    for (const item of items) {
+      for (const target of item.paths) {
+        block(item, env.scanPolicy.removalReason(target));
+      }
+      completed.add(item.id);
+    }
+    if (
+      env.kind === 'host' &&
+      context.scanner === 'global' &&
+      options.onScanPhase
+    ) {
+      const result = await options.onScanPhase(
+        sealReport({
+          schema: REPORT_SCHEMA,
+          tool: 'disk-space-saviour',
+          createdAt: new Date(options.now()).toISOString(),
+          host: { env: env.id, label: env.label, platform: env.platform },
+          options: reportOptions(options),
+          environments: [envDescriptor(env, 0, [])],
+          items: filterItems(items, options, [], env.path),
+          totals: tierTotals(items),
+          errors: [],
+          docker: null,
+        })
+      );
+      stopped ||= result === false;
+    }
+    return !stopped;
+  };
+  context.ready = ready;
   let incomplete = false;
   for (const name of options.scanners) {
     checkSignal(env.signal);
+    context.scanner = name;
     stage(name);
     const scanner = SCANNERS[name];
     if (!scanner) {
@@ -233,25 +334,31 @@ async function scanEnvironmentScoped(env, options, extra, found, stage) {
     const started = Date.now();
     const previous = new Set(found.keys());
     try {
-      const entries = await scanner(context);
+      const entries = await env.timings.measure(`scanner:${name}`, () =>
+        scanner(context)
+      );
       for (const item of entries) {
         context.emit(item);
       }
+      await ready(entries.filter((item) => !completed.has(item.id)));
     } catch (error) {
       checkSignal(env.signal);
       incomplete = true;
       for (const [id, item] of found) {
-        if (!previous.has(id)) {
+        if (![previous, completed].some((set) => set.has(id))) {
           block(item, `scan incomplete: ${error.message}`);
         }
       }
       errors.push({ env: env.id, scanner: name, message: error.message });
     }
     trace('scanner', name, 'in', env.id, `${Date.now() - started}ms`);
+    if (stopped) {
+      env.scanStatus = 'goal-reached';
+      return [...found.values()];
+    }
   }
   stage('storage');
   const items = [...found.values()];
-  await attributeStorage(env, items);
   for (const item of items) {
     for (const target of item.paths) {
       block(item, env.scanPolicy.removalReason(target));
@@ -385,13 +492,18 @@ function envDescriptor(env, depth, chain) {
     hint: env.probeHint ?? null,
     scanStatus: env.scanStatus ?? null,
     layerSource: env.writableLayer?.source ?? null,
+    timings: env.timings?.toJSON() ?? [],
   };
 }
 
 async function withDisk(env, descriptor) {
   const target = env.platform === 'win32' ? env.currentHome : '/';
   const disk = await env.diskUsage(target).catch(() => null);
-  return { ...descriptor, disk: disk ? { path: target, ...disk } : null };
+  return {
+    ...descriptor,
+    disk: disk ? { path: target, ...disk } : null,
+    swap: await readSwap(env),
+  };
 }
 
 async function dockerSection(env, options, errors) {
@@ -479,9 +591,19 @@ async function scanScoped(input) {
   const env = input.env ?? new LocalEnv();
   const startedAt = options.now();
   const errors = [];
+  env.scanMetrics = {
+    filesSized: 0,
+    directoriesSized: 0,
+    indexHits: 0,
+    sizeCacheHits: 0,
+  };
+  env.scanIndex = new ScanIndex(options.cacheDir);
+  env.platformIndex = options.platformIndex;
+  await env.scanIndex.load();
   resetWritableLayers(env.executor);
   env.writableLayer = null;
   env.scanPolicy = null;
+  env.timings = new PhaseTimer();
   await configureScanPolicy(env, options);
   await configureContainerHost(
     env,
@@ -492,7 +614,12 @@ async function scanScoped(input) {
     options.host === false
       ? []
       : await scanEnvironment(env, options, { roots: options.roots, errors });
-  const docker = await dockerSection(env, options, errors);
+  const docker =
+    env.scanStatus === 'goal-reached'
+      ? null
+      : await timed(env.timings, 'docker', () =>
+          dockerSection(env, options, errors)
+        );
   blockMountedHostItems(hostItems, docker?.bindMounts ?? [], env.path);
   const items = filterItems(
     [...hostItems, ...(docker?.items ?? [])],
@@ -504,7 +631,10 @@ async function scanScoped(input) {
   for (const inner of docker?.environments ?? []) {
     environments.push(inner);
   }
-  environments[0] = await withDisk(env, environments[0]);
+  environments[0] = await timed(env.timings, 'disk', () =>
+    withDisk(env, environments[0])
+  );
+  environments[0].timings = env.timings.toJSON();
   const report = buildReport({
     options,
     env,
@@ -515,6 +645,10 @@ async function scanScoped(input) {
     startedAt,
   });
   reportEnvironments.set(report, env);
+  await env.scanIndex.save();
+  await authenticateReport(report, options.cacheDir);
+  sealReport(report);
+  await saveScan(report, options.cacheDir);
   return report;
 }
 
@@ -524,6 +658,7 @@ function reportOptions(options) {
   delete serializable.env;
   delete serializable.signal;
   delete serializable.onScanProgress;
+  delete serializable.onScanPhase;
   return serializable;
 }
 

package/src/scanners/global.js

@@ -14,6 +14,7 @@ import { expandGlobPaths, expandRulePath, isWithin } from '../paths.js';
 import { olderThan, scanTimeBusy, resolvePaths } from './common.js';
 import { configuredPattern } from '../env/cache-roots.js';
 import { TOOLCHAIN_CACHE_RULES } from '../rules/toolchains.js';
+import { timed } from '../timings.js';
 
 export const GLOBAL_RULES = [
   ...CACHE_RULES,
@@ -135,6 +136,10 @@ function ageFilter(context, rule) {
   return () => true;
 }
 
+function requiresMtime(rule) {
+  return rule.mtime ?? (rule.scope === 'tmp' || Boolean(rule.minAge));
+}
+
 /** Keep the latest installed revision of each browser and platform family. */
 function olderVersions(env, rule, paths, stats) {
   if (!rule.versionPattern) {
@@ -181,6 +186,7 @@ async function buildItem(context, rule, base, group) {
     path: paths[0],
     paths,
     bytes: group.reduce((sum, entry) => sum + entry.usage.bytes, 0),
+    sizeEstimated: group.some((entry) => entry.usage.estimated),
     newestMtimeMs: Math.max(...group.map((e) => e.usage.newestMtimeMs ?? 0)),
     tier: rule.tier ?? 'safe',
     reason: rule.minAge
@@ -237,9 +243,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       patterns: rulePatterns(env, rule, base),
     }))
   );
-  const matches = await expandGlobPaths(
-    env,
-    planned.flatMap((entry) => entry.patterns)
+  const matches = await timed(context.timings, 'global:expand', () =>
+    expandGlobPaths(
+      env,
+      planned.flatMap((entry) => entry.patterns)
+    )
   );
   const typedPaths = planned
     .filter(
@@ -248,7 +256,9 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
     .flatMap(({ patterns }) =>
       patterns.flatMap((pattern) => matches.get(pattern) ?? [])
     );
-  const stats = await env.statMany([...new Set(typedPaths)]);
+  const stats = await timed(context.timings, 'global:stat', () =>
+    env.statMany([...new Set(typedPaths)])
+  );
   const found = [];
   const seen = new Set();
   for (const { rule, base, patterns } of planned) {
@@ -258,7 +268,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
 
     const accepted = [];
     for (const target of candidates) {
-      if (!(await acceptedPath(env, rule, target, stats))) {
+      if (
+        !(await timed(context.timings, 'global:accept', () =>
+          acceptedPath(env, rule, target, stats)
+        ))
+      ) {
         continue;
       }
       accepted.push(target);
@@ -269,22 +283,43 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       found.push({ rule, base, paths });
     }
   }
-  const usages = await env.usageMany(found.flatMap((entry) => entry.paths));
-  await resolvePaths(
-    context,
-    found.flatMap((entry) => entry.paths)
+  await timed(context.timings, 'global:resolve', () =>
+    resolvePaths(
+      context,
+      found.flatMap((entry) => entry.paths)
+    )
   );
   const items = scanItems(context);
+  // Deliver cheap native measurements before rules needing recursive activity
+  // timestamps, so an emergency goal can avoid the expensive work entirely.
+  found.sort(
+    (a, b) => Number(requiresMtime(a.rule)) - Number(requiresMtime(b.rule))
+  );
+  // The report drops items below `minSize`; skipping them here saves their
+  // liveness checks (Telegram media alone can hold 100k small files). In a
+  // container the writable layer re-measures sizes later, so all are kept.
+  const large = (entry) =>
+    Boolean(env.writableLayer) ||
+    entry.usage.bytes >= (options.minSizeBytes ?? 0);
   for (const { rule, base, paths } of found) {
+    const usages = await timed(context.timings, 'global:measure', () =>
+      env.usageMany(paths, { mtime: requiresMtime(rule), cache: true })
+    );
     const accept = ageFilter(context, rule);
     const measured = paths
       .map((target) => ({ path: target, usage: usages.get(target) }))
       .filter((entry) => entry.usage && accept(entry.usage));
     const groups = rule.aggregate
       ? [measured]
-      : measured.map((entry) => [entry]);
+      : measured.filter(large).map((entry) => [entry]);
     for (const group of groups.filter((g) => g.length > 0)) {
-      items.push(await buildItem(context, rule, base, group));
+      const item = await timed(context.timings, 'global:item', () =>
+        buildItem(context, rule, base, group)
+      );
+      items.push(item);
+      if ((await context.ready?.([item])) === false) {
+        return items;
+      }
     }
   }
   return items;