disk-space-saviour 0.0.0-stage → 0.12.0

package/src/scanners/global.js

@@ -0,0 +1,217 @@
+/**
+ * Global per-user caches (ecosystem download and compilation caches, IDE and
+ * browser caches), system package caches, and leftovers in temporary
+ * directories. See ../rules/ecosystems.js and ../rules/other.js.
+ */
+
+import { CACHE_RULES } from '../rules/ecosystems.js';
+import { OTHER_RULES } from '../rules/other.js';
+import { compareVersions } from '../rules/versions.js';
+import { block, makeItem } from '../items.js';
+import { expandGlobPaths, expandRulePath } from '../paths.js';
+import { olderThan, scanTimeBusy } from './common.js';
+
+export const GLOBAL_RULES = [...CACHE_RULES, ...OTHER_RULES];
+
+function scopeBases(rule, homes, tmpDirs) {
+  const scope = rule.scope ?? 'home';
+  if (scope === 'tmp') {
+    return tmpDirs.map((tmp) => ({ home: null, tmp }));
+  }
+  if (scope === 'system') {
+    return [{ home: null, tmp: null }];
+  }
+  return homes.map((home) => ({ home, tmp: null }));
+}
+
+function rulePatterns(env, rule, base) {
+  const patterns = [];
+  for (const pattern of rule.paths) {
+    const usesVars = /\{(?!TMP\})[A-Z_]+\}/.test(pattern);
+    if (usesVars && base.home !== env.currentHome) {
+      continue;
+    }
+    const expanded = expandRulePath(pattern, {
+      home: base.home ?? '/',
+      vars: { ...env.vars, TMP: base.tmp ?? undefined },
+      pathApi: env.path,
+    });
+    if (expanded) {
+      patterns.push(expanded);
+    }
+  }
+  return patterns;
+}
+
+/**
+ * The tool's own clean command, when it can run for this cache.
+ * @returns {Promise<{action: object|null, blocker: string|null}>}
+ */
+async function nativeAction(context, rule, base, paths) {
+  const { env, options } = context;
+  const native = rule.native;
+  const removal = { type: 'remove', paths };
+  if (!native || options.noNative) {
+    return { action: removal, blocker: null };
+  }
+  const usable =
+    (rule.scope === 'system' || base.home === env.currentHome) &&
+    (await env.which(native.tool)) &&
+    (!native.root || (await env.isRoot()));
+  if (usable) {
+    return {
+      action: {
+        type: 'command',
+        argv: native.argv,
+        measure: paths,
+      },
+      blocker: null,
+    };
+  }
+  if (native.required) {
+    return {
+      action: null,
+      blocker: `only \`${native.argv.join(' ')}\` can clean this safely, and it is unavailable`,
+    };
+  }
+  return { action: removal, blocker: null };
+}
+
+function ageFilter(context, rule) {
+  const { options } = context;
+  if (rule.minAge === 'stale') {
+    return (usage) =>
+      olderThan(context, usage.newestMtimeMs, options.staleAgeMs);
+  }
+  if (rule.minAge === 'inactive') {
+    return (usage) =>
+      olderThan(context, usage.newestMtimeMs, options.inactiveMs);
+  }
+  return () => true;
+}
+
+/** Keep the latest installed revision of each browser and platform family. */
+async function olderVersions(env, rule, paths) {
+  if (!rule.versionPattern) {
+    return paths;
+  }
+  const groups = new Map();
+  for (const target of paths) {
+    if ((await env.stat(target))?.type !== 'dir') {
+      continue;
+    }
+    const match = rule.versionPattern.exec(env.path.basename(target));
+    if (!match) {
+      continue;
+    }
+    const family = `${env.path.dirname(target)}\0${match.groups.family ?? ''}`;
+    groups.set(family, [
+      ...(groups.get(family) ?? []),
+      { target, version: match.groups.version },
+    ]);
+  }
+  const old = new Set();
+  for (const entries of groups.values()) {
+    entries.sort((a, b) => compareVersions(b.version, a.version));
+    for (const entry of entries.slice(1)) {
+      old.add(entry.target);
+    }
+  }
+  return paths.filter((target) => old.has(target));
+}
+
+async function buildItem(context, rule, base, group) {
+  const { env } = context;
+  const paths = group.map((entry) => entry.path);
+  const { action, blocker } = await nativeAction(context, rule, base, paths);
+  const scope = rule.scope ?? 'home';
+  const item = makeItem(env, {
+    rule: rule.id,
+    kind: scope === 'tmp' ? 'leftover' : 'cache',
+    ecosystem: rule.ecosystem,
+    description: rule.description,
+    path: paths[0],
+    paths,
+    bytes: group.reduce((sum, entry) => sum + entry.usage.bytes, 0),
+    newestMtimeMs: Math.max(...group.map((e) => e.usage.newestMtimeMs ?? 0)),
+    tier: rule.tier ?? 'safe',
+    reason: rule.minAge
+      ? `untouched for longer than the ${rule.minAge === 'stale' ? 'activity' : 'inactivity'} window`
+      : 're-downloaded or re-created on demand',
+    action: action ?? { type: 'none' },
+    checks: {
+      busy: rule.busy ?? [],
+      cwd: scope === 'tmp' ? paths[0] : null,
+      mtime: scope === 'tmp' || Boolean(rule.minAge),
+    },
+  });
+  block(item, blocker);
+  await addBlockers(context, rule, item);
+  return item;
+}
+
+async function addBlockers(context, rule, item) {
+  const { env, git } = context;
+  if (rule.scope === 'system' && !(await env.isRoot())) {
+    block(item, 'system location, run as root to clean it');
+  }
+  if (rule.git) {
+    for (const target of item.paths) {
+      for (const reason of await git.treeBlockers(target)) {
+        block(item, reason);
+      }
+    }
+  }
+  const busy = scanTimeBusy(context, item);
+  if (busy) {
+    block(item, `busy: ${busy}`);
+  }
+}
+
+/**
+ * Scans global cache and leftover locations.
+ * @returns {Promise<object[]>} report items
+ */
+export async function scanGlobal(context, { rules = GLOBAL_RULES } = {}) {
+  const { env, options } = context;
+  const homes = options.homes ?? (await env.homeDirs());
+  const tmpDirs = options.tmpDirs ?? (await env.tmpDirs());
+  const planned = rules.flatMap((rule) =>
+    scopeBases(rule, homes, tmpDirs).map((base) => ({
+      rule,
+      base,
+      patterns: rulePatterns(env, rule, base),
+    }))
+  );
+  const matches = await expandGlobPaths(
+    env,
+    planned.flatMap((entry) => entry.patterns)
+  );
+  const found = [];
+  const seen = new Set();
+  for (const { rule, base, patterns } of planned) {
+    const candidates = [
+      ...new Set(patterns.flatMap((pattern) => matches.get(pattern) ?? [])),
+    ].filter((target) => !seen.has(target));
+    candidates.forEach((target) => seen.add(target));
+    const paths = await olderVersions(env, rule, candidates);
+    if (paths.length > 0) {
+      found.push({ rule, base, paths });
+    }
+  }
+  const usages = await env.usageMany(found.flatMap((entry) => entry.paths));
+  const items = [];
+  for (const { rule, base, paths } of found) {
+    const accept = ageFilter(context, rule);
+    const measured = paths
+      .map((target) => ({ path: target, usage: usages.get(target) }))
+      .filter((entry) => entry.usage && accept(entry.usage));
+    const groups = rule.aggregate
+      ? [measured]
+      : measured.map((entry) => [entry]);
+    for (const group of groups.filter((g) => g.length > 0)) {
+      items.push(await buildItem(context, rule, base, group));
+    }
+  }
+  return items;
+}

package/src/scanners/projects.js

@@ -0,0 +1,335 @@
+/**
+ * Per-project dependency, build and cache directories (node_modules, target,
+ * .venv, ...) found by walking the scan roots.
+ *
+ * A directory matches a rule when its name matches and the rule's markers
+ * are present. Generic names (`build`, `lib`, `target`, `bin`) that fail
+ * every rule are searched again below, with the remaining depth, so a
+ * non-matching `build` directory never hides real projects inside it.
+ *
+ * Cargo `target` directories additionally yield `cargo-superseded` items:
+ * older build hashes that a newer build of the same unit replaced (see
+ * `rust.js`).
+ */
+
+import { ECOSYSTEMS, PROJECT_RULES } from '../rules/ecosystems.js';
+import { block, makeItem } from '../items.js';
+import { isWithin, matchesGlob } from '../paths.js';
+import { listMany, olderThan, scanTimeBusy } from './common.js';
+import { analyzeRustProfile, findRustProfiles } from './rust.js';
+
+const ECOSYSTEM_NAMES = new Map(ECOSYSTEMS.map((e) => [e.id, e.name]));
+
+function anyMatch(entries, globs) {
+  return (entries ?? []).some((entry) =>
+    globs.some((glob) => matchesGlob(entry.name, glob))
+  );
+}
+
+function depthBelow(env, roots, target) {
+  const root = roots
+    .filter((candidate) => isWithin(target, candidate, env.path))
+    .sort((a, b) => b.length - a.length)[0];
+  return env.path.relative(root, target).split(/[\\/]/).filter(Boolean).length;
+}
+
+function ruleAccepts(env, rule, candidate, selfListing, grandListing) {
+  if (rule.markers && !anyMatch(candidate.siblings, rule.markers)) {
+    return false;
+  }
+  if (rule.selfMarkers && !anyMatch(selfListing, rule.selfMarkers)) {
+    return false;
+  }
+  if (rule.parentName) {
+    if (env.path.basename(candidate.parent) !== rule.parentName) {
+      return false;
+    }
+    if (rule.parentMarkers && !anyMatch(grandListing, rule.parentMarkers)) {
+      return false;
+    }
+  }
+  if (
+    rule.lockfiles &&
+    !anyMatch(
+      rule.parentName ? grandListing : candidate.siblings,
+      rule.lockfiles
+    )
+  ) {
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Pairs every found directory with the first rule it satisfies (or null).
+ */
+async function matchRules(env, found, rules) {
+  const options = found.map((candidate) => ({
+    candidate,
+    rules: rules.filter((rule) =>
+      rule.names.some((glob) => matchesGlob(candidate.name, glob))
+    ),
+  }));
+  const selfDirs = options
+    .filter((o) => o.rules.some((rule) => rule.selfMarkers))
+    .map((o) => o.candidate.path);
+  const grandDirs = options
+    .filter((o) =>
+      o.rules.some(
+        (rule) => rule.parentName === env.path.basename(o.candidate.parent)
+      )
+    )
+    .map((o) => env.path.dirname(o.candidate.parent));
+  const selfListings = await listMany(env, selfDirs);
+  const grandListings = await listMany(env, [...new Set(grandDirs)]);
+  return options.map(({ candidate, rules: possible }) => ({
+    candidate,
+    rule:
+      possible.find((rule) =>
+        ruleAccepts(
+          env,
+          rule,
+          candidate,
+          selfListings.get(candidate.path),
+          grandListings.get(env.path.dirname(candidate.parent))
+        )
+      ) ?? null,
+  }));
+}
+
+async function rootCandidates(env, roots) {
+  const candidates = [];
+  for (const root of roots) {
+    const parent = env.path.dirname(root);
+    if (parent === root) {
+      continue;
+    }
+    const siblings = await env.list(parent);
+    candidates.push({
+      path: root,
+      name: env.path.basename(root),
+      parent,
+      siblings,
+    });
+  }
+  return candidates;
+}
+
+/**
+ * Finds every rule match below the roots.
+ * @returns {Promise<Array<{candidate: object, rule: object}>>}
+ */
+export async function findProjectDirs(env, roots, options = {}) {
+  const rules = options.rules ?? PROJECT_RULES;
+  const maxDepth = options.maxDepth ?? 6;
+  const globs = [...new Set(rules.flatMap((rule) => rule.names))];
+  const matches = [];
+  if (options.includeRoots) {
+    const self = await matchRules(env, await rootCandidates(env, roots), rules);
+    matches.push(...self.filter((match) => match.rule));
+  }
+  let frontier = new Map([[maxDepth, roots]]);
+  while (frontier.size > 0) {
+    const next = new Map();
+    for (const [depth, dirs] of frontier) {
+      const found = await env.findDirs(dirs, { globs, maxDepth: depth });
+      for (const match of await matchRules(env, found, rules)) {
+        if (match.rule) {
+          matches.push(match);
+          continue;
+        }
+        const remaining = depth - depthBelow(env, dirs, match.candidate.path);
+        if (remaining > 0) {
+          next.set(remaining, [
+            ...(next.get(remaining) ?? []),
+            match.candidate.path,
+          ]);
+        }
+      }
+    }
+    frontier = next;
+  }
+  return matches;
+}
+
+function siblingActivity(candidate, globs) {
+  return (candidate.siblings ?? [])
+    .filter(
+      (entry) =>
+        entry.name !== candidate.name &&
+        !globs.some((glob) => matchesGlob(entry.name, glob))
+    )
+    .reduce((newest, entry) => Math.max(newest, entry.mtimeMs ?? 0), 0);
+}
+
+function projectTier(context, rule, lastActivityMs) {
+  if (rule.kind === 'cache') {
+    return { tier: 'safe', reason: 'cache, re-created on demand' };
+  }
+  const { inactiveMs } = context.options;
+  if (olderThan(context, lastActivityMs, inactiveMs)) {
+    return {
+      tier: 'moderate',
+      reason: 'project inactive, re-created by the build or install step',
+    };
+  }
+  return {
+    tier: 'aggressive',
+    reason: 'project recently active, re-created by the build or install step',
+  };
+}
+
+async function addBlockers(context, item, requireClean) {
+  for (const reason of await context.git.pathBlockers(item.path, {
+    requireClean,
+  })) {
+    block(item, reason);
+  }
+  const busy = scanTimeBusy(context, item);
+  if (busy) {
+    block(item, `busy: ${busy}`);
+  }
+}
+
+async function projectActivity(context, project, candidate, usage, globs) {
+  const { env, git } = context;
+  const repo = await git.findRepoRoot(project);
+  const lastCommitMs = repo ? (await git.repoState(repo)).lastCommitMs : 0;
+  context.projectUsage ??= new Map();
+  if (!context.projectUsage.has(project)) {
+    context.projectUsage.set(project, await env.usage(project));
+  }
+  const projectUsage = context.projectUsage.get(project);
+  return Math.max(
+    usage?.newestMtimeMs ?? 0,
+    projectUsage?.newestMtimeMs ?? 0,
+    siblingActivity(candidate, globs),
+    lastCommitMs
+  );
+}
+
+async function projectItem(context, { candidate, rule }, usage, globs) {
+  const { env, options } = context;
+  const project = rule.parentName
+    ? env.path.dirname(candidate.parent)
+    : candidate.parent;
+  const lastActivityMs = await projectActivity(
+    context,
+    project,
+    candidate,
+    usage,
+    globs
+  );
+  const item = makeItem(env, {
+    rule: rule.id,
+    kind: rule.kind,
+    ecosystem: rule.ecosystem,
+    description: `${ECOSYSTEM_NAMES.get(rule.ecosystem)} ${rule.description}`,
+    path: candidate.path,
+    bytes: usage?.bytes ?? 0,
+    newestMtimeMs: usage?.newestMtimeMs ?? 0,
+    lastActivityMs,
+    project,
+    lockfiles: rule.lockfiles ?? null,
+    ...projectTier(context, rule, lastActivityMs),
+    checks: { busy: rule.busy ?? [], cwd: project, mtime: true },
+  });
+  await addBlockers(
+    context,
+    item,
+    item.tier !== 'safe' && !options.allowDirtyRepos
+  );
+  return item;
+}
+
+const RUST_PARTS = {
+  library: {
+    rule: 'cargo-superseded',
+    description: (part) =>
+      `superseded cargo artifacts (${part.removed} old units, ${part.kept} kept)`,
+    reason: 'older build hashes replaced by a newer build of the same unit',
+  },
+  leaf: {
+    rule: 'cargo-superseded-leaf',
+    description: (part) =>
+      `idle cargo incremental caches and superseded test/example binaries (${part.removed} old, ${part.kept} kept)`,
+    reason:
+      'nothing links against them; removable while cargo runs, files in use are skipped',
+  },
+};
+
+/**
+ * One item per part and profile. The library part is busy while cargo or
+ * rustc runs in the project. The leaf part ignores running tools and drops
+ * only the paths that a process holds open (`checks.perPath`).
+ */
+async function rustItems(context, targetItem, rule) {
+  const { env, liveness, options, now } = context;
+  const items = [];
+  for (const profile of await findRustProfiles(env, targetItem.path)) {
+    const result = await analyzeRustProfile(env, profile, {
+      staleAgeMs: options.staleAgeMs,
+      now,
+    });
+    for (const [name, spec] of Object.entries(RUST_PARTS)) {
+      const part = result[name];
+      const leaf = name === 'leaf';
+      const open = leaf
+        ? (liveness?.openPathsOf(part.entries.map((e) => e.path)) ?? new Map())
+        : new Map();
+      const entries = part.entries.filter((entry) => !open.has(entry.path));
+      if (entries.length === 0) {
+        continue;
+      }
+      const item = makeItem(env, {
+        rule: spec.rule,
+        kind: 'build',
+        ecosystem: 'rust',
+        description: spec.description(part),
+        path: profile,
+        paths: entries.map((entry) => entry.path),
+        bytes: entries.reduce((sum, entry) => sum + entry.bytes, 0),
+        newestMtimeMs: Math.max(...entries.map((entry) => entry.mtimeMs)),
+        project: targetItem.project,
+        tier: 'safe',
+        reason: spec.reason,
+        parentId: targetItem.id,
+        recheck: { type: 'rust', profile, part: name },
+        checks: leaf
+          ? { busy: [], cwd: targetItem.project, mtime: true, perPath: true }
+          : { busy: rule.busy, cwd: targetItem.project, mtime: true },
+      });
+      await addBlockers(context, item, false);
+      items.push(item);
+    }
+  }
+  return items;
+}
+
+/**
+ * Scans project trees below `context.options.roots`.
+ * @returns {Promise<object[]>} report items
+ */
+export async function scanProjects(context, { rules = PROJECT_RULES } = {}) {
+  const { env, options } = context;
+  const matches = await findProjectDirs(env, options.roots, {
+    rules,
+    maxDepth: options.maxDepth,
+    includeRoots: options.includeRoots,
+  });
+  const usages = await env.usageMany(matches.map((m) => m.candidate.path));
+  const globs = [...new Set(rules.flatMap((rule) => rule.names))];
+  const items = [];
+  for (const match of matches) {
+    const usage = usages.get(match.candidate.path);
+    if (!usage) {
+      continue;
+    }
+    const item = await projectItem(context, match, usage, globs);
+    items.push(item);
+    if (match.rule.id === 'cargo-target') {
+      items.push(...(await rustItems(context, item, match.rule)));
+    }
+  }
+  return items;
+}