disk-space-saviour 0.17.0 → 0.19.0

package/src/scanners/global.js

@@ -2,7 +2,8 @@ import { scanItems } from '../env/scope.js';
 /**
  * Global per-user caches (ecosystem download and compilation caches, IDE and
  * browser caches), system package caches, and leftovers in temporary
- * directories. See ../rules/ecosystems.js and ../rules/other.js.
+ * directories. See ../rules/ecosystems.js, ../rules/other.js and
+ * ../rules/downloads.js.
  */
 
 import { CACHE_RULES } from '../rules/ecosystems.js';
@@ -14,12 +15,15 @@ import { expandGlobPaths, expandRulePath, isWithin } from '../paths.js';
 import { olderThan, scanTimeBusy, resolvePaths } from './common.js';
 import { configuredPattern } from '../env/cache-roots.js';
 import { TOOLCHAIN_CACHE_RULES } from '../rules/toolchains.js';
+import { DOWNLOAD_RULES } from '../rules/downloads.js';
+import { timed } from '../timings.js';
 
 export const GLOBAL_RULES = [
   ...CACHE_RULES,
   ...expandChromiumCaches(OTHER_RULES),
   ...APP_RULES,
   ...TOOLCHAIN_CACHE_RULES,
+  ...DOWNLOAD_RULES,
 ];
 
 function scopeBases(rule, homes, tmpDirs) {
@@ -242,9 +246,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       patterns: rulePatterns(env, rule, base),
     }))
   );
-  const matches = await expandGlobPaths(
-    env,
-    planned.flatMap((entry) => entry.patterns)
+  const matches = await timed(context.timings, 'global:expand', () =>
+    expandGlobPaths(
+      env,
+      planned.flatMap((entry) => entry.patterns)
+    )
   );
   const typedPaths = planned
     .filter(
@@ -253,7 +259,9 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
     .flatMap(({ patterns }) =>
       patterns.flatMap((pattern) => matches.get(pattern) ?? [])
     );
-  const stats = await env.statMany([...new Set(typedPaths)]);
+  const stats = await timed(context.timings, 'global:stat', () =>
+    env.statMany([...new Set(typedPaths)])
+  );
   const found = [];
   const seen = new Set();
   for (const { rule, base, patterns } of planned) {
@@ -263,7 +271,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
 
     const accepted = [];
     for (const target of candidates) {
-      if (!(await acceptedPath(env, rule, target, stats))) {
+      if (
+        !(await timed(context.timings, 'global:accept', () =>
+          acceptedPath(env, rule, target, stats)
+        ))
+      ) {
         continue;
       }
       accepted.push(target);
@@ -274,9 +286,11 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
       found.push({ rule, base, paths });
     }
   }
-  await resolvePaths(
-    context,
-    found.flatMap((entry) => entry.paths)
+  await timed(context.timings, 'global:resolve', () =>
+    resolvePaths(
+      context,
+      found.flatMap((entry) => entry.paths)
+    )
   );
   const items = scanItems(context);
   // Deliver cheap native measurements before rules needing recursive activity
@@ -284,20 +298,27 @@ export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
   found.sort(
     (a, b) => Number(requiresMtime(a.rule)) - Number(requiresMtime(b.rule))
   );
+  // The report drops items below `minSize`; skipping them here saves their
+  // liveness checks (Telegram media alone can hold 100k small files). In a
+  // container the writable layer re-measures sizes later, so all are kept.
+  const large = (entry) =>
+    Boolean(env.writableLayer) ||
+    entry.usage.bytes >= (options.minSizeBytes ?? 0);
   for (const { rule, base, paths } of found) {
-    const usages = await env.usageMany(paths, {
-      mtime: requiresMtime(rule),
-      cache: true,
-    });
+    const usages = await timed(context.timings, 'global:measure', () =>
+      env.usageMany(paths, { mtime: requiresMtime(rule), cache: true })
+    );
     const accept = ageFilter(context, rule);
     const measured = paths
       .map((target) => ({ path: target, usage: usages.get(target) }))
       .filter((entry) => entry.usage && accept(entry.usage));
     const groups = rule.aggregate
       ? [measured]
-      : measured.map((entry) => [entry]);
+      : measured.filter(large).map((entry) => [entry]);
     for (const group of groups.filter((g) => g.length > 0)) {
-      const item = await buildItem(context, rule, base, group);
+      const item = await timed(context.timings, 'global:item', () =>
+        buildItem(context, rule, base, group)
+      );
       items.push(item);
       if ((await context.ready?.([item])) === false) {
         return items;

package/src/scanners/projects.js

@@ -1,3 +1,5 @@
+import { performance } from 'node:perf_hooks';
+
 import { scanItems } from '../env/scope.js';
 /**
  * Per-project dependency, build and cache directories (node_modules, target,
@@ -16,8 +18,9 @@ import { scanItems } from '../env/scope.js';
 import { ECOSYSTEMS, PROJECT_RULES } from '../rules/ecosystems.js';
 import { block, makeItem } from '../items.js';
 import { isWithin, matchesGlob } from '../paths.js';
-import { listMany, olderThan, scanTimeBusy } from './common.js';
+import { listMany, scanTimeBusy } from './common.js';
 import { analyzeRustProfile, findRustProfiles } from './rust.js';
+import { timed } from '../timings.js';
 
 const ECOSYSTEM_NAMES = new Map(ECOSYSTEMS.map((e) => [e.id, e.name]));
 
@@ -183,106 +186,38 @@ export async function findProjectDirs(env, roots, options = {}) {
   return matches;
 }
 
-function siblingActivity(candidate, globs) {
-  return (candidate.siblings ?? [])
-    .filter(
-      (entry) =>
-        entry.name !== candidate.name &&
-        !globs.some((glob) => matchesGlob(entry.name, glob))
-    )
-    .reduce((newest, entry) => Math.max(newest, entry.mtimeMs ?? 0), 0);
-}
-
-function projectTier(context, rule, lastActivityMs) {
-  if (rule.kind === 'cache') {
-    return { tier: 'safe', reason: 'cache, re-created on demand' };
-  }
-  const { inactiveMs } = context.options;
-  if (olderThan(context, lastActivityMs, inactiveMs)) {
-    return {
-      tier: rule.id === 'node-modules' ? 'safe' : 'moderate',
-      reason: 'project inactive, re-created by the build or install step',
-    };
-  }
-  return {
-    tier: 'aggressive',
-    reason: 'project recently active, re-created by the build or install step',
-  };
-}
-
-/** Re-evaluate activity before cleaning a reused project plan. */
-export async function currentProjectTier(context, item) {
-  const rule = PROJECT_RULES.find((entry) => entry.id === item.rule);
-  if (!rule || rule.kind === 'cache') {
-    return item.tier;
-  }
-  const globs = [...new Set(PROJECT_RULES.flatMap((entry) => entry.names))];
-  const parent = context.env.path.dirname(item.path);
-  const candidate = {
-    name: context.env.path.basename(item.path),
-    siblings: await context.env.list(parent),
-  };
-  context.git.invalidate();
-  const activity = await projectActivity(
-    context,
-    item.project,
-    candidate,
-    { newestMtimeMs: item.newestMtimeMs },
-    globs
-  );
-  return projectTier(context, rule, activity).tier;
+/**
+ * Project caches, build outputs and dependencies are all regenerable, so how
+ * recently the project was used does not matter: only liveness (a tool
+ * running in the project, open files, recent writes) and Git safety block
+ * them.
+ */
+function projectTier(rule) {
+  return rule.kind === 'cache'
+    ? { tier: 'safe', reason: 'cache, re-created on demand' }
+    : { tier: 'safe', reason: 're-created by the build or install step' };
 }
 
 async function addBlockers(context, item, requireClean) {
-  for (const reason of await context.git.pathBlockers(item.path, {
-    requireClean,
-  })) {
+  const reasons = await timed(context.timings, 'projects:git', () =>
+    context.git.pathBlockers(item.path, { requireClean })
+  );
+  for (const reason of reasons) {
     block(item, reason);
   }
-  const busy = await scanTimeBusy(context, item);
+  const busy = await timed(context.timings, 'projects:busy', () =>
+    scanTimeBusy(context, item)
+  );
   if (busy) {
     block(item, `busy: ${busy}`);
   }
 }
 
-async function projectActivity(context, project, candidate, usage, globs) {
-  const { env, git } = context;
-  const repo = await git.findRepoRoot(project);
-  const lastCommitMs = repo ? (await git.repoState(repo)).lastCommitMs : 0;
-  context.projectUsage ??= new Map();
-  if (!context.projectUsage.has(project)) {
-    context.projectUsage.set(
-      project,
-      env.projectActivity
-        ? await env.projectActivity(
-            project,
-            globs,
-            context.now - context.options.inactiveMs
-          )
-        : (await env.usage(project))?.newestMtimeMs
-    );
-  }
-  const projectUsage = context.projectUsage.get(project);
-  return Math.max(
-    usage?.newestMtimeMs ?? 0,
-    projectUsage ?? 0,
-    siblingActivity(candidate, globs),
-    lastCommitMs
-  );
-}
-
-async function projectItem(context, { candidate, rule }, usage, globs) {
+async function projectItem(context, { candidate, rule }, usage) {
   const { env } = context;
   const project = rule.parentName
     ? env.path.dirname(candidate.parent)
     : candidate.parent;
-  const lastActivityMs = await projectActivity(
-    context,
-    project,
-    candidate,
-    usage,
-    globs
-  );
   const item = makeItem(env, {
     rule: rule.id,
     kind: rule.kind,
@@ -292,10 +227,9 @@ async function projectItem(context, { candidate, rule }, usage, globs) {
     bytes: usage?.bytes ?? 0,
     sizeEstimated: Boolean(usage?.estimated),
     newestMtimeMs: usage?.newestMtimeMs ?? 0,
-    lastActivityMs,
     project,
     lockfiles: rule.lockfiles ?? null,
-    ...projectTier(context, rule, lastActivityMs),
+    ...projectTier(rule),
     checks: { busy: rule.busy ?? [], cwd: project, mtime: true },
   });
   await addBlockers(context, item, false);
@@ -397,29 +331,51 @@ async function rustItems(context, targetItem, rule) {
  */
 export async function scanProjects(context, { rules = PROJECT_RULES } = {}) {
   const { env, options } = context;
-  const globs = [...new Set(rules.flatMap((rule) => rule.names))];
   const items = scanItems(context);
-  await findProjectDirs(env, options.roots, {
-    rules,
-    maxDepth: options.maxDepth,
-    includeRoots: options.includeRoots,
-    onMatches: async (matches) => {
-      const usages = await env.usageMany(
-        matches.map((match) => match.candidate.path),
-        { cache: true }
-      );
-      for (const match of matches) {
-        const usage = usages.get(match.candidate.path);
-        if (!usage) {
-          continue;
+  const started = performance.now();
+  // Matches are measured while the walk continues; `projects:walk` excludes
+  // that time, which `projects:measure`, `projects:git` and `projects:busy`
+  // record.
+  let streamedMs = 0;
+  try {
+    await findProjectDirs(env, options.roots, {
+      rules,
+      maxDepth: options.maxDepth,
+      includeRoots: options.includeRoots,
+      onMatches: async (matches) => {
+        const streamStarted = performance.now();
+        try {
+          await projectMatches(context, matches, items);
+        } finally {
+          streamedMs += performance.now() - streamStarted;
         }
-        const item = await projectItem(context, match, usage, globs);
-        items.push(item);
-        if (match.rule.id === 'cargo-target') {
-          items.push(...(await rustItems(context, item, match.rule)));
-        }
-      }
-    },
-  });
+      },
+    });
+  } finally {
+    context.timings?.record(
+      'projects:walk',
+      performance.now() - started - streamedMs
+    );
+  }
   return items;
 }
+
+async function projectMatches(context, matches, items) {
+  const usages = await timed(context.timings, 'projects:measure', () =>
+    context.env.usageMany(
+      matches.map((match) => match.candidate.path),
+      { cache: true }
+    )
+  );
+  for (const match of matches) {
+    const usage = usages.get(match.candidate.path);
+    if (!usage) {
+      continue;
+    }
+    const item = await projectItem(context, match, usage);
+    items.push(item);
+    if (match.rule.id === 'cargo-target') {
+      items.push(...(await rustItems(context, item, match.rule)));
+    }
+  }
+}

package/src/scanners/versions.js

@@ -6,7 +6,9 @@ import { scanItems } from '../env/scope.js';
  * Kept: the manager's default and alias targets, versions pinned by any
  * project below the scan roots, the newest install of each group, and
  * anything a running process executes from (checked by the liveness probe).
- * Everything else is reported as `moderate`.
+ * Everything else is reported as `moderate`, and blocked while it holds
+ * data that reinstalling the version would not bring back (pyenv virtual
+ * environments, globally installed npm packages).
  */
 
 import {
@@ -171,6 +173,98 @@ function newestInstalls(env, rule, installs, usages) {
   return newest;
 }
 
+/**
+ * Follows a chain of manager aliases (`default` -> `lts/iron` -> `v20.20.2`).
+ * An LTS alias also keeps its release line: nvm records the newest remote
+ * release there, which may not be the one installed.
+ * @returns {Promise<string[]>}
+ */
+async function resolveAlias(env, aliasDir, ref) {
+  const refs = [ref];
+  let current = ref;
+  for (let hop = 0; hop < 8 && !/^v?\d/.test(current); hop++) {
+    if (current.split('/').some((part) => part === '..' || part === '')) {
+      break;
+    }
+    const text = await env.readText(env.path.join(aliasDir, current), 4096);
+    const next = (text ?? '').split(/\s+/).find(Boolean);
+    if (!next || refs.includes(next)) {
+      break;
+    }
+    refs.push(next);
+    current = next;
+  }
+  if (ref.startsWith('lts/') && /^v?\d/.test(current)) {
+    refs.push(current.replace(/^v?(\d+).*$/, 'v$1'));
+  }
+  return refs;
+}
+
+async function resolveAliases(env, rule, home, refs) {
+  if (!rule.aliasDir) {
+    return refs;
+  }
+  const aliasDir = expandRulePath(configuredPattern(env, home, rule.aliasDir), {
+    home,
+    pathApi: env.path,
+  });
+  const resolved = [];
+  for (const ref of refs) {
+    resolved.push(...(await resolveAlias(env, aliasDir, ref)));
+  }
+  return [...new Set(resolved)];
+}
+
+/**
+ * Installs that the refs keep. With `refsPickNewest` a ref keeps only the
+ * newest install it matches, as nvm resolves `20` to the newest 20.x.
+ * @returns {Set<string>}
+ */
+function referencedInstalls(rule, installs, refs, versionOf) {
+  const kept = new Set();
+  for (const ref of refs) {
+    const matching = installs
+      .filter((dir) => versionMatches(versionOf(dir), ref))
+      .sort((a, b) => compareVersions(versionOf(b), versionOf(a)));
+    for (const dir of rule.refsPickNewest ? matching.slice(0, 1) : matching) {
+      kept.add(dir);
+    }
+  }
+  return kept;
+}
+
+/**
+ * Global packages a user installed into a version (`npm install -g`) are
+ * lost with it. Packages bundled with the runtime and caches that are
+ * installed again on demand (`rule.globalPackages.regenerable`) are not.
+ */
+async function blockGlobalPackages(env, rule, dir, item) {
+  if (!rule.globalPackages) {
+    return;
+  }
+  const names = [];
+  const root = env.path.join(dir, rule.globalPackages.dir);
+  for (const entry of await env.list(root)) {
+    if (entry.name.startsWith('.')) {
+      continue;
+    }
+    if (!entry.name.startsWith('@')) {
+      names.push(entry.name);
+      continue;
+    }
+    const scoped = await env.list(entry.path);
+    names.push(...scoped.map((child) => `${entry.name}/${child.name}`));
+  }
+  const own = names
+    .filter((name) => !rule.globalPackages.regenerable(name))
+    .sort();
+  if (own.length > 0) {
+    const shown = own.slice(0, 5).join(', ');
+    const more = own.length > 5 ? ` and ${own.length - 5} more` : '';
+    block(item, `has globally installed packages: ${shown}${more}`);
+  }
+}
+
 async function blockPyenvEnvironments(env, rule, dir, item) {
   if (rule.id !== 'pyenv-python') {
     return;
@@ -189,17 +283,21 @@ async function ruleItems(context, rule, home, pins) {
   if (rule.protectAll) {
     return [];
   }
-  const refs = [...(await managerRefs(env, rule, home, installs)), ...pins];
+  const refs = await resolveAliases(env, rule, home, [
+    ...(await managerRefs(env, rule, home, installs)),
+    ...pins,
+  ]);
   const usages = await env.usageMany(installs, {
     cache: !context.options.freshSizes,
   });
   await resolvePaths(context, installs);
   const newest = newestInstalls(env, rule, installs, usages);
   const versionOf = rule.versionOf ?? ((dir) => env.path.basename(dir));
+  const referenced = referencedInstalls(rule, installs, refs, versionOf);
   const items = scanItems(context);
   for (const dir of installs) {
     const version = versionOf(dir);
-    if (newest.has(dir) || refs.some((ref) => versionMatches(version, ref))) {
+    if (newest.has(dir) || referenced.has(dir)) {
       continue;
     }
     const usage = usages.get(dir) ?? {};
@@ -222,6 +320,7 @@ async function ruleItems(context, rule, home, pins) {
           : null,
     });
     await blockPyenvEnvironments(env, rule, dir, item);
+    await blockGlobalPackages(env, rule, dir, item);
     const busy = await scanTimeBusy(context, item);
     if (busy) {
       block(item, `busy: ${busy}`);

package/src/timings.js

@@ -0,0 +1,54 @@
+import { performance } from 'node:perf_hooks';
+
+/**
+ * Wall-clock phase timings.
+ *
+ * Collected on every run, not only with `--verbose`, so reports and audit
+ * logs show which phase was slow. A phase that throws or is aborted (for
+ * example by the scan budget) still records the time it consumed.
+ */
+
+export class PhaseTimer {
+  constructor() {
+    this.phases = new Map();
+  }
+
+  /**
+   * Runs `fn` and adds its duration to `name`.
+   * @template T
+   * @param {string} name
+   * @param {() => Promise<T>|T} fn
+   * @returns {Promise<T>}
+   */
+  async measure(name, fn) {
+    const started = performance.now();
+    try {
+      return await fn();
+    } finally {
+      this.record(name, performance.now() - started);
+    }
+  }
+
+  record(name, ms) {
+    const phase = this.phases.get(name) ?? { phase: name, ms: 0, count: 0 };
+    phase.ms += ms;
+    phase.count++;
+    this.phases.set(name, phase);
+  }
+
+  /**
+   * @returns {Array<{phase: string, ms: number, count: number}>} slowest first
+   */
+  toJSON() {
+    return [...this.phases.values()]
+      .map((phase) => ({ ...phase, ms: Math.round(phase.ms) }))
+      .sort((a, b) => b.ms - a.ms);
+  }
+}
+
+/**
+ * Measures `fn` when `timer` exists, otherwise just runs it.
+ */
+export function timed(timer, name, fn) {
+  return timer ? timer.measure(name, fn) : fn();
+}