disk-space-saviour 0.16.0 → 0.18.0

package/src/scanners/projects.js

@@ -1,3 +1,5 @@
+import { performance } from 'node:perf_hooks';
+
 import { scanItems } from '../env/scope.js';
 /**
  * Per-project dependency, build and cache directories (node_modules, target,
@@ -16,8 +18,9 @@ import { scanItems } from '../env/scope.js';
 import { ECOSYSTEMS, PROJECT_RULES } from '../rules/ecosystems.js';
 import { block, makeItem } from '../items.js';
 import { isWithin, matchesGlob } from '../paths.js';
-import { listMany, olderThan, scanTimeBusy } from './common.js';
+import { listMany, scanTimeBusy } from './common.js';
 import { analyzeRustProfile, findRustProfiles } from './rust.js';
+import { timed } from '../timings.js';
 
 const ECOSYSTEM_NAMES = new Map(ECOSYSTEMS.map((e) => [e.id, e.name]));
 
@@ -65,6 +68,14 @@ function ruleAccepts(env, rule, candidate, selfListing, grandListing) {
  * Pairs every found directory with the first rule it satisfies (or null).
  */
 async function matchRules(env, found, rules) {
+  // Refresh markers and activity; the discovery index only supplies hints.
+  const parents = await listMany(env, [
+    ...new Set(found.map((candidate) => candidate.parent)),
+  ]);
+  found = found.map((candidate) => ({
+    ...candidate,
+    siblings: parents.get(candidate.parent) ?? [],
+  }));
   const options = found.map((candidate) => ({
     candidate,
     rules: rules.filter((rule) =>
@@ -116,6 +127,11 @@ async function rootCandidates(env, roots) {
   return candidates;
 }
 
+async function indexedMatches(env, roots, globs, maxDepth, rules) {
+  const candidates = (await env.indexedDirs?.(roots, globs, maxDepth)) ?? [];
+  return matchRules(env, candidates, rules);
+}
+
 /**
  * Finds every rule match below the roots.
  * @returns {Promise<Array<{candidate: object, rule: object}>>}
@@ -125,18 +141,34 @@ export async function findProjectDirs(env, roots, options = {}) {
   const maxDepth = options.maxDepth ?? 6;
   const globs = [...new Set(rules.flatMap((rule) => rule.names))];
   const matches = [];
+  const seen = new Set();
+  const accept = (match) => {
+    if (!match.rule || seen.has(match.candidate.path)) {
+      return;
+    }
+    seen.add(match.candidate.path);
+    matches.push(match);
+    return match;
+  };
+  const indexed = await indexedMatches(env, roots, globs, maxDepth, rules);
+  await options.onMatches?.(indexed.map(accept).filter(Boolean));
   if (options.includeRoots) {
     const self = await matchRules(env, await rootCandidates(env, roots), rules);
-    matches.push(...self.filter((match) => match.rule));
+    const accepted = self.map(accept).filter(Boolean);
+    await options.onMatches?.(accepted);
   }
   let frontier = new Map([[maxDepth, roots]]);
   while (frontier.size > 0) {
     const next = new Map();
     for (const [depth, dirs] of frontier) {
       const found = await env.findDirs(dirs, { globs, maxDepth: depth });
+      const accepted = [];
       for (const match of await matchRules(env, found, rules)) {
         if (match.rule) {
-          matches.push(match);
+          const fresh = accept(match);
+          if (fresh) {
+            accepted.push(fresh);
+          }
           continue;
         }
         const remaining = depth - depthBelow(env, dirs, match.candidate.path);
@@ -147,80 +179,45 @@ export async function findProjectDirs(env, roots, options = {}) {
           ]);
         }
       }
+      await options.onMatches?.(accepted);
     }
     frontier = next;
   }
   return matches;
 }
 
-function siblingActivity(candidate, globs) {
-  return (candidate.siblings ?? [])
-    .filter(
-      (entry) =>
-        entry.name !== candidate.name &&
-        !globs.some((glob) => matchesGlob(entry.name, glob))
-    )
-    .reduce((newest, entry) => Math.max(newest, entry.mtimeMs ?? 0), 0);
-}
-
-function projectTier(context, rule, lastActivityMs) {
-  if (rule.kind === 'cache') {
-    return { tier: 'safe', reason: 'cache, re-created on demand' };
-  }
-  const { inactiveMs } = context.options;
-  if (olderThan(context, lastActivityMs, inactiveMs)) {
-    return {
-      tier: rule.id === 'node-modules' ? 'safe' : 'moderate',
-      reason: 'project inactive, re-created by the build or install step',
-    };
-  }
-  return {
-    tier: 'aggressive',
-    reason: 'project recently active, re-created by the build or install step',
-  };
+/**
+ * Project caches, build outputs and dependencies are all regenerable, so how
+ * recently the project was used does not matter: only liveness (a tool
+ * running in the project, open files, recent writes) and Git safety block
+ * them.
+ */
+function projectTier(rule) {
+  return rule.kind === 'cache'
+    ? { tier: 'safe', reason: 'cache, re-created on demand' }
+    : { tier: 'safe', reason: 're-created by the build or install step' };
 }
 
 async function addBlockers(context, item, requireClean) {
-  for (const reason of await context.git.pathBlockers(item.path, {
-    requireClean,
-  })) {
+  const reasons = await timed(context.timings, 'projects:git', () =>
+    context.git.pathBlockers(item.path, { requireClean })
+  );
+  for (const reason of reasons) {
     block(item, reason);
   }
-  const busy = await scanTimeBusy(context, item);
+  const busy = await timed(context.timings, 'projects:busy', () =>
+    scanTimeBusy(context, item)
+  );
   if (busy) {
     block(item, `busy: ${busy}`);
   }
 }
 
-async function projectActivity(context, project, candidate, usage, globs) {
-  const { env, git } = context;
-  const repo = await git.findRepoRoot(project);
-  const lastCommitMs = repo ? (await git.repoState(repo)).lastCommitMs : 0;
-  context.projectUsage ??= new Map();
-  if (!context.projectUsage.has(project)) {
-    context.projectUsage.set(project, await env.usage(project));
-  }
-  const projectUsage = context.projectUsage.get(project);
-  return Math.max(
-    usage?.newestMtimeMs ?? 0,
-    projectUsage?.newestMtimeMs ?? 0,
-    siblingActivity(candidate, globs),
-    lastCommitMs
-  );
-}
-
-async function projectItem(context, { candidate, rule }, usage, globs) {
+async function projectItem(context, { candidate, rule }, usage) {
   const { env } = context;
   const project = rule.parentName
     ? env.path.dirname(candidate.parent)
     : candidate.parent;
-  const lastActivityMs = await projectActivity(
-    context,
-    project,
-    candidate,
-    usage,
-    globs
-  );
   const item = makeItem(env, {
     rule: rule.id,
     kind: rule.kind,
@@ -228,11 +225,11 @@ async function projectItem(context, { candidate, rule }, usage, globs) {
     description: `${ECOSYSTEM_NAMES.get(rule.ecosystem)} ${rule.description}`,
     path: candidate.path,
     bytes: usage?.bytes ?? 0,
+    sizeEstimated: Boolean(usage?.estimated),
     newestMtimeMs: usage?.newestMtimeMs ?? 0,
-    lastActivityMs,
     project,
     lockfiles: rule.lockfiles ?? null,
-    ...projectTier(context, rule, lastActivityMs),
+    ...projectTier(rule),
     checks: { busy: rule.busy ?? [], cwd: project, mtime: true },
   });
   await addBlockers(context, item, false);
@@ -264,7 +261,7 @@ async function rustItems(context, targetItem, rule) {
   const { env, liveness, options, now } = context;
   const items = scanItems(context);
   const semverPath = env.path.join(targetItem.path, 'semver-checks');
-  const semverUsage = await env.usage(semverPath);
+  const semverUsage = await env.usage(semverPath, { cache: true });
   if (semverUsage) {
     const semver = makeItem(env, {
       rule: 'cargo-semver-checks-cache',
@@ -273,6 +270,7 @@ async function rustItems(context, targetItem, rule) {
       description: 'cargo-semver-checks baseline builds',
       path: semverPath,
       bytes: semverUsage.bytes,
+      sizeEstimated: Boolean(semverUsage.estimated),
       newestMtimeMs: semverUsage.newestMtimeMs,
       project: targetItem.project,
       parentId: targetItem.id,
@@ -333,24 +331,51 @@ async function rustItems(context, targetItem, rule) {
  */
 export async function scanProjects(context, { rules = PROJECT_RULES } = {}) {
   const { env, options } = context;
-  const matches = await findProjectDirs(env, options.roots, {
-    rules,
-    maxDepth: options.maxDepth,
-    includeRoots: options.includeRoots,
-  });
-  const usages = await env.usageMany(matches.map((m) => m.candidate.path));
-  const globs = [...new Set(rules.flatMap((rule) => rule.names))];
   const items = scanItems(context);
+  const started = performance.now();
+  // Matches are measured while the walk continues; `projects:walk` excludes
+  // that time, which `projects:measure`, `projects:git` and `projects:busy`
+  // record.
+  let streamedMs = 0;
+  try {
+    await findProjectDirs(env, options.roots, {
+      rules,
+      maxDepth: options.maxDepth,
+      includeRoots: options.includeRoots,
+      onMatches: async (matches) => {
+        const streamStarted = performance.now();
+        try {
+          await projectMatches(context, matches, items);
+        } finally {
+          streamedMs += performance.now() - streamStarted;
+        }
+      },
+    });
+  } finally {
+    context.timings?.record(
+      'projects:walk',
+      performance.now() - started - streamedMs
+    );
+  }
+  return items;
+}
+
+async function projectMatches(context, matches, items) {
+  const usages = await timed(context.timings, 'projects:measure', () =>
+    context.env.usageMany(
+      matches.map((match) => match.candidate.path),
+      { cache: true }
+    )
+  );
   for (const match of matches) {
     const usage = usages.get(match.candidate.path);
     if (!usage) {
       continue;
     }
-    const item = await projectItem(context, match, usage, globs);
+    const item = await projectItem(context, match, usage);
     items.push(item);
     if (match.rule.id === 'cargo-target') {
       items.push(...(await rustItems(context, item, match.rule)));
     }
   }
-  return items;
 }

package/src/scanners/versions.js

@@ -6,7 +6,9 @@ import { scanItems } from '../env/scope.js';
  * Kept: the manager's default and alias targets, versions pinned by any
  * project below the scan roots, the newest install of each group, and
  * anything a running process executes from (checked by the liveness probe).
- * Everything else is reported as `moderate`.
+ * Everything else is reported as `moderate`, and blocked while it holds
+ * data that reinstalling the version would not bring back (pyenv virtual
+ * environments, globally installed npm packages).
  */
 
 import {
@@ -171,6 +173,98 @@ function newestInstalls(env, rule, installs, usages) {
   return newest;
 }
 
+/**
+ * Follows a chain of manager aliases (`default` -> `lts/iron` -> `v20.20.2`).
+ * An LTS alias also keeps its release line: nvm records the newest remote
+ * release there, which may not be the one installed.
+ * @returns {Promise<string[]>}
+ */
+async function resolveAlias(env, aliasDir, ref) {
+  const refs = [ref];
+  let current = ref;
+  for (let hop = 0; hop < 8 && !/^v?\d/.test(current); hop++) {
+    if (current.split('/').some((part) => part === '..' || part === '')) {
+      break;
+    }
+    const text = await env.readText(env.path.join(aliasDir, current), 4096);
+    const next = (text ?? '').split(/\s+/).find(Boolean);
+    if (!next || refs.includes(next)) {
+      break;
+    }
+    refs.push(next);
+    current = next;
+  }
+  if (ref.startsWith('lts/') && /^v?\d/.test(current)) {
+    refs.push(current.replace(/^v?(\d+).*$/, 'v$1'));
+  }
+  return refs;
+}
+
+async function resolveAliases(env, rule, home, refs) {
+  if (!rule.aliasDir) {
+    return refs;
+  }
+  const aliasDir = expandRulePath(configuredPattern(env, home, rule.aliasDir), {
+    home,
+    pathApi: env.path,
+  });
+  const resolved = [];
+  for (const ref of refs) {
+    resolved.push(...(await resolveAlias(env, aliasDir, ref)));
+  }
+  return [...new Set(resolved)];
+}
+
+/**
+ * Installs that the refs keep. With `refsPickNewest` a ref keeps only the
+ * newest install it matches, as nvm resolves `20` to the newest 20.x.
+ * @returns {Set<string>}
+ */
+function referencedInstalls(rule, installs, refs, versionOf) {
+  const kept = new Set();
+  for (const ref of refs) {
+    const matching = installs
+      .filter((dir) => versionMatches(versionOf(dir), ref))
+      .sort((a, b) => compareVersions(versionOf(b), versionOf(a)));
+    for (const dir of rule.refsPickNewest ? matching.slice(0, 1) : matching) {
+      kept.add(dir);
+    }
+  }
+  return kept;
+}
+
+/**
+ * Global packages a user installed into a version (`npm install -g`) are
+ * lost with it. Packages bundled with the runtime and caches that are
+ * installed again on demand (`rule.globalPackages.regenerable`) are not.
+ */
+async function blockGlobalPackages(env, rule, dir, item) {
+  if (!rule.globalPackages) {
+    return;
+  }
+  const names = [];
+  const root = env.path.join(dir, rule.globalPackages.dir);
+  for (const entry of await env.list(root)) {
+    if (entry.name.startsWith('.')) {
+      continue;
+    }
+    if (!entry.name.startsWith('@')) {
+      names.push(entry.name);
+      continue;
+    }
+    const scoped = await env.list(entry.path);
+    names.push(...scoped.map((child) => `${entry.name}/${child.name}`));
+  }
+  const own = names
+    .filter((name) => !rule.globalPackages.regenerable(name))
+    .sort();
+  if (own.length > 0) {
+    const shown = own.slice(0, 5).join(', ');
+    const more = own.length > 5 ? ` and ${own.length - 5} more` : '';
+    block(item, `has globally installed packages: ${shown}${more}`);
+  }
+}
+
 async function blockPyenvEnvironments(env, rule, dir, item) {
   if (rule.id !== 'pyenv-python') {
     return;
@@ -189,26 +283,33 @@ async function ruleItems(context, rule, home, pins) {
   if (rule.protectAll) {
     return [];
   }
-  const refs = [...(await managerRefs(env, rule, home, installs)), ...pins];
-  const usages = await env.usageMany(installs);
+  const refs = await resolveAliases(env, rule, home, [
+    ...(await managerRefs(env, rule, home, installs)),
+    ...pins,
+  ]);
+  const usages = await env.usageMany(installs, {
+    cache: !context.options.freshSizes,
+  });
   await resolvePaths(context, installs);
   const newest = newestInstalls(env, rule, installs, usages);
   const versionOf = rule.versionOf ?? ((dir) => env.path.basename(dir));
+  const referenced = referencedInstalls(rule, installs, refs, versionOf);
   const items = scanItems(context);
   for (const dir of installs) {
     const version = versionOf(dir);
-    if (newest.has(dir) || refs.some((ref) => versionMatches(version, ref))) {
+    if (newest.has(dir) || referenced.has(dir)) {
       continue;
     }
-    const usage = usages.get(dir);
+    const usage = usages.get(dir) ?? {};
     const item = makeItem(env, {
       rule: rule.id,
       kind: 'toolchain',
       ecosystem: rule.ecosystem,
       description: `${rule.description} ${version}`,
       path: dir,
-      bytes: usage?.bytes ?? 0,
-      newestMtimeMs: usage?.newestMtimeMs ?? 0,
+      bytes: usage.bytes ?? 0,
+      sizeEstimated: Boolean(usage.estimated),
+      newestMtimeMs: usage.newestMtimeMs ?? 0,
       tier: 'moderate',
       reason: `not the ${rule.manager} default, not pinned by a scanned project, not the newest`,
       checks: { busy: rule.busy ?? [], cwd: dir, mtime: true },
@@ -219,6 +320,7 @@ async function ruleItems(context, rule, home, pins) {
           : null,
     });
     await blockPyenvEnvironments(env, rule, dir, item);
+    await blockGlobalPackages(env, rule, dir, item);
     const busy = await scanTimeBusy(context, item);
     if (busy) {
       block(item, `busy: ${busy}`);

package/src/swap.js

@@ -0,0 +1,69 @@
+/** Swap is a capacity diagnostic, never a deletion candidate. */
+import { parseSize } from './units.js';
+
+export function parseSwapUsage(text) {
+  const match =
+    /total\s*=\s*([\d.]+[KMGT]?B?)\s+used\s*=\s*([\d.]+[KMGT]?B?)\s+free\s*=\s*([\d.]+[KMGT]?B?)/i.exec(
+      text
+    );
+  return match
+    ? {
+        total: parseSize(match[1]),
+        used: parseSize(match[2]),
+        free: parseSize(match[3]),
+      }
+    : null;
+}
+
+export function parseMemoryHolders(text) {
+  return text
+    .split('\n')
+    .map((line) =>
+      /^\s*(\d+)\s+(.+?)\s+([\d.]+[KMGT]?B?[+-]?)\s+([\d.]+[KMGT]?B?[+-]?)\s*$/i.exec(
+        line
+      )
+    )
+    .filter(Boolean)
+    .map((match) => ({
+      pid: Number(match[1]),
+      command: match[2],
+      memoryBytes: parseSize(match[3].replace(/[+-]$/, '')),
+      compressedBytes: parseSize(match[4].replace(/[+-]$/, '')),
+    }))
+    .sort((a, b) => b.memoryBytes - a.memoryBytes)
+    .slice(0, 10);
+}
+
+export async function readSwap(env) {
+  if (env.kind !== 'host' || env.platform !== 'darwin') {
+    return null;
+  }
+  try {
+    const usage = await env.run(['sysctl', 'vm.swapusage']);
+    const swap = usage.code === 0 ? parseSwapUsage(usage.stdout) : null;
+    if (!swap) {
+      return null;
+    }
+    const top = await env.run([
+      'top',
+      '-l',
+      '1',
+      '-o',
+      'mem',
+      '-n',
+      '10',
+      '-stats',
+      'pid,command,mem,cmprs',
+    ]);
+    return {
+      ...swap,
+      holders: top.code === 0 ? parseMemoryHolders(top.stdout) : [],
+      attribution:
+        'Largest resident/compressed memory consumers; macOS does not expose per-process swap ownership here.',
+      advice:
+        'If Docker Desktop is consuming memory, restart it after saving running work or lower its VM memory limit. Swap files are managed by macOS; dss never deletes them.',
+    };
+  } catch {
+    return null;
+  }
+}

package/src/timings.js

@@ -0,0 +1,54 @@
+import { performance } from 'node:perf_hooks';
+
+/**
+ * Wall-clock phase timings.
+ *
+ * Collected on every run, not only with `--verbose`, so reports and audit
+ * logs show which phase was slow. A phase that throws or is aborted (for
+ * example by the scan budget) still records the time it consumed.
+ */
+
+export class PhaseTimer {
+  constructor() {
+    this.phases = new Map();
+  }
+
+  /**
+   * Runs `fn` and adds its duration to `name`.
+   * @template T
+   * @param {string} name
+   * @param {() => Promise<T>|T} fn
+   * @returns {Promise<T>}
+   */
+  async measure(name, fn) {
+    const started = performance.now();
+    try {
+      return await fn();
+    } finally {
+      this.record(name, performance.now() - started);
+    }
+  }
+
+  record(name, ms) {
+    const phase = this.phases.get(name) ?? { phase: name, ms: 0, count: 0 };
+    phase.ms += ms;
+    phase.count++;
+    this.phases.set(name, phase);
+  }
+
+  /**
+   * @returns {Array<{phase: string, ms: number, count: number}>} slowest first
+   */
+  toJSON() {
+    return [...this.phases.values()]
+      .map((phase) => ({ ...phase, ms: Math.round(phase.ms) }))
+      .sort((a, b) => b.ms - a.ms);
+  }
+}
+
+/**
+ * Measures `fn` when `timer` exists, otherwise just runs it.
+ */
+export function timed(timer, name, fn) {
+  return timer ? timer.measure(name, fn) : fn();
+}