disk-space-saviour 0.16.0 → 0.18.0

package/src/env/local.js

@@ -48,6 +48,40 @@ export function parseLsofCwds(output) {
   return cwds;
 }
 
+/**
+ * Absolute directories of a `PATH` value.
+ * @param {string} value
+ * @returns {string[]}
+ */
+export function searchPathDirs(value) {
+  return (value ?? '').split(':').filter((dir) => dir.startsWith('/'));
+}
+
+/**
+ * `PATH` directories by pid from `ps -E -ww -axo pid=,command=` output,
+ * which appends the environment of the user's own processes to their
+ * arguments. A value ends at the first space, so a directory whose name
+ * contains one is not tracked.
+ * @param {string} output
+ * @returns {Map<number, string[]>}
+ */
+export function parsePsSearchPaths(output) {
+  const paths = new Map();
+  for (const line of output.split('\n')) {
+    const match = /^\s*(\d+)\s/.exec(line);
+    if (!match) {
+      continue;
+    }
+    const dirs = [...line.matchAll(/(?:^|\s)PATH=(\S*)/g)].flatMap((found) =>
+      searchPathDirs(found[1])
+    );
+    if (dirs.length > 0) {
+      paths.set(Number(match[1]), [...new Set(dirs)]);
+    }
+  }
+  return paths;
+}
+
 /**
  * Environment adapter for the machine running dss, backed by node:fs.
  *
@@ -64,6 +98,9 @@ import { CACHE_VARIABLES } from './cache-roots.js';
 
 import { hostExecutor } from '../exec.js';
 import { matchesGlob } from '../paths.js';
+import { mapLimit } from './parallel.js';
+import { directoryFingerprint, SIZE_MAX_AGE } from '../scan-state.js';
+import { indexedDirectories } from './platform-index.js';
 
 // Directories read concurrently while measuring usage.
 const WALK_BATCH = 64;
@@ -252,6 +289,7 @@ export class LocalEnv {
       size: stats.size,
       bytes: diskBytes(stats),
       mtimeMs: stats.mtimeMs,
+      ctimeMs: stats.ctimeMs,
       inode: `${stats.dev}:${stats.ino}`,
       uid: stats.uid,
       gid: stats.gid,
@@ -295,31 +333,36 @@ export class LocalEnv {
     return heads;
   }
 
-  async list(dir) {
+  async list(dir, { discovery = false } = {}) {
     let names;
     try {
-      names = await (this.options.fileFs ?? fsp).readdir(dir);
+      names = await (this.options.fileFs ?? fsp).readdir(dir, {
+        withFileTypes: true,
+      });
     } catch (error) {
       this.recordAccessError(dir, error);
       return [];
     }
-    const entries = await Promise.all(
-      names.map(async (name) => {
-        const full = path.join(dir, name);
-        const stats = await this.#lstat(full);
-        if (!stats) {
-          return null;
-        }
-        return {
-          name,
-          path: full,
-          type: entryType(stats),
-          size: stats.size,
-          bytes: diskBytes(stats),
-          mtimeMs: stats.mtimeMs,
-        };
-      })
-    );
+    const entries = await mapLimit(names, 32, async (entry) => {
+      const name = typeof entry === 'string' ? entry : entry.name;
+      const full = path.join(dir, name);
+      if (discovery && typeof entry !== 'string' && !entry.isDirectory()) {
+        return { name, path: full, type: entryType(entry), mtimeMs: 0 };
+      }
+      const stats = await this.#lstat(full);
+      if (!stats) {
+        return null;
+      }
+      return {
+        name,
+        path: full,
+        type: entryType(stats),
+        size: stats.size,
+        bytes: diskBytes(stats),
+        mtimeMs: stats.mtimeMs,
+        device: stats.dev,
+      };
+    });
     return entries.filter(Boolean);
   }
 
@@ -328,14 +371,15 @@ export class LocalEnv {
    * counted a single time, symlinks not followed. Also reports the newest
    * file mtime.
    */
-  async usage(target) {
-    return (await this.usageMany([target])).get(target) ?? null;
+  async usage(target, options) {
+    return (await this.usageMany([target], options)).get(target) ?? null;
   }
 
   async statMany(targets) {
-    const entries = await Promise.all(
-      targets.map(async (target) => [target, await this.stat(target)])
-    );
+    const entries = await mapLimit(targets, 32, async (target) => [
+      target,
+      await this.stat(target),
+    ]);
     return new Map(entries.filter(([, stat]) => stat));
   }
 
@@ -362,6 +406,9 @@ export class LocalEnv {
       apparentBytes += stats.size;
       if (!stats.isDirectory()) {
         files++;
+        if (this.scanMetrics) {
+          this.scanMetrics.filesSized++;
+        }
         newestMtimeMs = Math.max(newestMtimeMs, stats.mtimeMs);
       }
     };
@@ -375,12 +422,10 @@ export class LocalEnv {
         errors++;
         return [];
       }
-      const children = await Promise.all(
-        names.map(async (name) => {
-          const full = path.join(dir, name);
-          return [full, await this.#lstat(full)];
-        })
-      );
+      const children = await mapLimit(names, 32, async (name) => {
+        const full = path.join(dir, name);
+        return [full, await this.#lstat(full)];
+      });
       const subdirs = [];
       for (const [full, stats] of children) {
         if (!stats) {
@@ -453,6 +498,10 @@ export class LocalEnv {
     return results;
   }
 
+  indexedDirs(roots, globs, maxDepth) {
+    return indexedDirectories(this, roots, globs, maxDepth);
+  }
+
   /**
    * Finds regular files whose basename matches one of `names`.
    * @returns {Promise<string[]>}
@@ -468,7 +517,14 @@ export class LocalEnv {
           results.push(entry.path);
         }
       }
-      return null;
+      return new Set([
+        'node_modules',
+        'target',
+        'vendor',
+        '.venv',
+        'venv',
+        ...this.skipNames,
+      ]);
     });
     return results;
   }
@@ -480,55 +536,185 @@ export class LocalEnv {
   async #walk(roots, maxDepth, skipNames, visit) {
     const skip = skipNames ? new Set(skipNames) : this.skipNames;
     const visited = new Set();
-    const queue = await Promise.all(
+    let queue = await Promise.all(
       roots.map(async (root) => [root, 0, await this.deviceId(root)])
     );
     while (queue.length > 0) {
       checkSignal(this.signal);
-      const [dir, depth, device] = queue.shift();
-      if (visited.has(dir)) {
-        continue;
-      }
-      visited.add(dir);
-      const entries = [];
-      for (const entry of await this.list(dir)) {
-        if (
-          entry.type !== 'dir' ||
-          (await this.deviceId(entry.path)) === device
-        ) {
-          entries.push(entry);
-        }
-      }
-      const pruned = visit(dir, entries) ?? new Set();
-      if (depth + 1 >= maxDepth) {
-        continue;
-      }
-      for (const entry of entries) {
-        if (
-          entry.type === 'dir' &&
-          !pruned.has(entry.name) &&
-          !skip.has(entry.name) &&
-          (await this.deviceId(entry.path)) === device
-        ) {
-          queue.push([entry.path, depth + 1, device]);
+      const current = queue;
+      queue = [];
+      const children = await mapLimit(
+        current,
+        8,
+        async ([dir, depth, device]) => {
+          if (visited.has(dir)) {
+            return [];
+          }
+          visited.add(dir);
+          const entries = [];
+          for (const entry of await this.discoveryList(dir)) {
+            if (
+              entry.type !== 'dir' ||
+              (entry.device ?? (await this.deviceId(entry.path))) === device
+            ) {
+              entries.push(entry);
+            }
+          }
+          const pruned = visit(dir, entries) ?? new Set();
+          if (depth + 1 >= maxDepth) {
+            return [];
+          }
+          const next = [];
+          for (const entry of entries) {
+            if (
+              entry.type === 'dir' &&
+              !pruned.has(entry.name) &&
+              !this.skipDiscoveryDirectory(dir, entry.name, skip) &&
+              (entry.device ?? (await this.deviceId(entry.path))) === device
+            ) {
+              next.push([entry.path, depth + 1, device]);
+            }
+          }
+          return next;
         }
+      );
+      queue = children.flat();
+    }
+  }
+
+  skipDiscoveryDirectory(dir, name, skip = this.skipNames) {
+    if (skip.has(name) || /\.photoslibrary$/i.test(name)) {
+      return true;
+    }
+    const homes = this.options.homes ?? [this.currentHome];
+    return (
+      (['Music', 'Pictures', 'Movies'].includes(name) && homes.includes(dir)) ||
+      (name === 'Volumes' && dir === this.path.parse(dir).root)
+    );
+  }
+
+  async discoveryStat(dir) {
+    let stats = await this.stat(dir);
+    // Explicit roots may be aliases (for example /tmp on macOS). Child links
+    // are never queued by the walker, so only an explicitly supplied root follows.
+    if (stats?.type === 'symlink') {
+      const resolved = await this.realPath(dir);
+      stats = resolved ? await this.stat(resolved) : null;
+    }
+    return stats;
+  }
+
+  async discoveryList(dir) {
+    checkSignal(this.signal);
+    const stats = await this.discoveryStat(dir);
+    if (!stats || stats.type !== 'dir') {
+      return [];
+    }
+    const key = `list:${dir}`;
+    const fingerprint = `${directoryFingerprint(stats)}:${JSON.stringify([this.scanPolicy?.exclude, this.scanPolicy?.pruned])}`;
+    let entries = this.scanIndex?.get(key, fingerprint);
+    if (entries) {
+      this.scanMetrics.indexHits++;
+    } else {
+      const errors = this.accessErrors?.size ?? 0;
+      entries = await this.list(dir, { discovery: true });
+      if ((this.accessErrors?.size ?? 0) === errors) {
+        this.scanIndex?.set(key, fingerprint, entries);
       }
     }
+    // Stored metadata is a discovery hint; never accept external paths from it.
+    const listing = entries
+      .filter(
+        (entry) => entry.name && this.path.basename(entry.name) === entry.name
+      )
+      .map((entry) => ({ ...entry, path: this.path.join(dir, entry.name) }))
+      .filter(
+        (entry) => !this.scanPolicy || this.scanPolicy.allows(entry.path)
+      );
+    // Mounts and link replacements need not change the parent's mtime.
+    return (
+      await mapLimit(listing, 32, async (entry) => {
+        if (entry.type !== 'dir') {
+          return entry;
+        }
+        const fresh = await this.stat(entry.path);
+        return fresh ? { ...entry, ...fresh } : null;
+      })
+    ).filter(Boolean);
   }
 
-  async usageMany(targets) {
-    const usages = await this.rawUsageMany(targets);
+  async usageMany(targets, options) {
+    const usages = await this.rawUsageMany(targets, options);
     return this.writableLayer
       ? this.writableLayer.measure(this, usages)
       : usages;
   }
 
-  async rawUsageMany(targets) {
-    const usages = new Map();
-    for (const target of targets) {
-      usages.set(target, await this.rawUsage(target));
+  async rawUsageMany(targets, { mtime = true, cache = false } = {}) {
+    const entries = await mapLimit([...new Set(targets)], 4, async (target) => [
+      target,
+      !this.writableLayer && cache
+        ? await this.cachedUsage(target, mtime)
+        : !mtime && !this.writableLayer
+          ? await this.quickUsage(target)
+          : await this.rawUsage(target),
+    ]);
+    return new Map(entries);
+  }
+
+  async cachedUsage(target, mtime) {
+    const stats = await this.stat(target);
+    if (!stats) {
+      return null;
     }
-    return usages;
+    const key = `usage:${mtime}:${target}`;
+    const fingerprint = `${directoryFingerprint(stats)}:${JSON.stringify([this.scanPolicy?.exclude, this.scanPolicy?.pruned])}`;
+    const cached = this.scanIndex?.get(key, fingerprint, SIZE_MAX_AGE);
+    if (cached) {
+      this.scanMetrics.sizeCacheHits++;
+      return { ...cached, estimated: true };
+    }
+    const usage = mtime
+      ? await this.rawUsage(target)
+      : await this.quickUsage(target);
+    if (usage && !usage.errors) {
+      this.scanIndex?.set(key, fingerprint, usage);
+    }
+    return usage;
+  }
+
+  async quickUsage(target) {
+    const stats = await this.stat(target);
+    if (!stats) {
+      return null;
+    }
+    // du cannot enforce path-glob exclusions or detect a nested mount beneath a candidate.
+    if (
+      this.platform === 'win32' ||
+      this.scanPolicy?.exclude.length ||
+      this.scanPolicy?.mounts.some(
+        (mount) =>
+          mount.path !== '/' && mount.path.startsWith(`${target}${path.sep}`)
+      )
+    ) {
+      return this.rawUsage(target);
+    }
+    const result = await this.run(['du', '-skx', target]);
+    const match = /^\s*(\d+)\s/.exec(result.stdout);
+    if (result.code !== 0 || !match) {
+      return this.rawUsage(target);
+    }
+    const usage = {
+      bytes: Number(match[1]) * 1024,
+      apparentBytes: Number(match[1]) * 1024,
+      newestMtimeMs: 0,
+      files: 0,
+      errors: 0,
+    };
+    if (this.scanMetrics) {
+      this.scanMetrics.directoriesSized++;
+    }
+    return usage;
   }
 
   async readLink(target) {
@@ -608,6 +794,15 @@ export class LocalEnv {
             ).stdout
           )
         : new Map();
+    // A process may start programs from its PATH at any time without holding
+    // them open (for example a release directory of an updated tool).
+    const searchPaths =
+      this.platform === 'darwin'
+        ? parsePsSearchPaths(
+            (await this.run(['ps', '-E', '-ww', '-axo', 'pid=,command=']))
+              .stdout
+          )
+        : new Map();
     return result.stdout
       .split('\n')
       .map((line) => /^\s*(\d+)\s+(.+)$/.exec(line))
@@ -617,6 +812,7 @@ export class LocalEnv {
         name: path.basename(m[2].trim()),
         cwd: cwds.get(Number(m[1])) ?? null,
         command: commands.get(Number(m[1])) ?? '',
+        searchPath: searchPaths.get(Number(m[1])) ?? [],
       }))
       .filter((proc) => !selfPids.has(proc.pid));
   }
@@ -650,6 +846,9 @@ export class LocalEnv {
       const cmdline = await io
         .readFile(`${base}/cmdline`, 'utf8')
         .catch(() => '');
+      const environ = await io
+        .readFile(`${base}/environ`, 'utf8')
+        .catch(() => '');
       processes.push({
         pid: Number(pid),
         ...(await procIdentity(base, io)),
@@ -660,6 +859,12 @@ export class LocalEnv {
         aliases: processAliases(exe, cmdline.split('\0')[0]),
         command: cmdline.replaceAll('\0', ' '),
         argv: cmdline.split('\0').filter(Boolean),
+        searchPath: searchPathDirs(
+          environ
+            .split('\0')
+            .find((entry) => entry.startsWith('PATH='))
+            ?.slice('PATH='.length)
+        ),
       });
     }
     return processes;

package/src/env/parallel.js

@@ -0,0 +1,23 @@
+/** Finite worker pool; settle all in-flight work before propagating failure. */
+export async function mapLimit(values, limit, operation) {
+  const results = new Array(values.length);
+  let cursor = 0;
+  let failure;
+  const worker = async () => {
+    while (!failure && cursor < values.length) {
+      const index = cursor++;
+      try {
+        results[index] = await operation(values[index], index);
+      } catch (error) {
+        failure ??= error;
+      }
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, values.length) }, worker)
+  );
+  if (failure) {
+    throw failure;
+  }
+  return results;
+}

package/src/env/platform-index.js

@@ -0,0 +1,87 @@
+/** OS index hits are hints; stale/missing indexes always fall back to discovery. */
+import { isWithin, matchesGlob } from '../paths.js';
+import { mapLimit } from './parallel.js';
+
+export async function indexedDirectories(env, roots, globs, maxDepth) {
+  if (!env.scanIndex?.directory || env.platformIndex === false) {
+    return [];
+  }
+  let hits = [];
+  try {
+    if (env.platform === 'darwin' && (await env.which('mdfind'))) {
+      const query = globs
+        .map((glob) => `kMDItemFSName == "${glob.replaceAll('"', '\\"')}"`)
+        .join(' || ');
+      for (const root of roots) {
+        const result = await env.run(['mdfind', '-0', '-onlyin', root, query]);
+        if (result.code === 0) {
+          hits.push(...result.stdout.split('\0').filter(Boolean));
+        }
+      }
+    } else if (env.platform === 'linux') {
+      const command = (await env.which('plocate'))
+        ? 'plocate'
+        : (await env.which('locate'))
+          ? 'locate'
+          : null;
+      if (command) {
+        const result = await env.run([command, '-0', '-e', '-b', ...globs]);
+        if (result.code === 0) {
+          hits = result.stdout.split('\0').filter(Boolean);
+        }
+      }
+    }
+  } catch {
+    // Budget/cancellation will be observed by the fallback walk too.
+  }
+  hits = [...new Set(hits)]
+    .filter((target) => {
+      const root = roots.find((candidate) =>
+        isWithin(target, candidate, env.path)
+      );
+      if (
+        !root ||
+        !env.path.isAbsolute(target) ||
+        !globs.some((glob) => matchesGlob(env.path.basename(target), glob))
+      ) {
+        return false;
+      }
+      const parts = env.path.relative(root, target).split(/[\\/]/);
+      return (
+        parts.length <= maxDepth &&
+        !parts
+          .slice(0, -1)
+          .some(
+            (name, index) =>
+              env.skipDiscoveryDirectory(
+                env.path.join(root, ...parts.slice(0, index)),
+                name
+              ) || globs.some((glob) => matchesGlob(name, glob))
+          )
+      );
+    })
+    .slice(0, 20000);
+  const candidates = await mapLimit(hits, 8, async (target) => {
+    const stat = await env.stat(target);
+    const root = roots.find((candidate) =>
+      isWithin(target, candidate, env.path)
+    );
+    if (stat?.type !== 'dir' || stat.device !== (await env.deviceId(root))) {
+      return null;
+    }
+    let parent = env.path.dirname(target);
+    while (parent !== root && isWithin(parent, root, env.path)) {
+      const ancestor = await env.stat(parent);
+      if (ancestor?.type !== 'dir' || ancestor.device !== stat.device) {
+        return null;
+      }
+      parent = env.path.dirname(parent);
+    }
+    return {
+      path: target,
+      name: env.path.basename(target),
+      parent: env.path.dirname(target),
+    };
+  });
+  return candidates.filter(Boolean);
+}

package/src/env/shell.js

@@ -405,7 +405,16 @@ export class ShellEnv {
   }
 
   findFiles(roots, { names, maxDepth = 6, skipNames } = {}) {
-    return this.#find(roots, 'f', names, maxDepth, skipNames);
+    return this.#find(roots, 'f', names, maxDepth, [
+      ...new Set([
+        ...(skipNames ?? this.skipNames),
+        'node_modules',
+        'target',
+        'vendor',
+        '.venv',
+        'venv',
+      ]),
+    ]);
   }
 
   /**

package/src/fast-clean.js

@@ -0,0 +1,39 @@
+/** Clean completed safe host findings while discovery continues. */
+import { clean } from './clean.js';
+import { scan } from './scan.js';
+
+export async function cleanDuringScan(options) {
+  const entries = [];
+  const health = [];
+  let failure;
+  const report = await scan({
+    ...options,
+    onScanPhase: async (phase) => {
+      try {
+        const audit = await clean(phase, {
+          ...options,
+          tier: 'safe',
+          priorEntries: entries,
+          priorHealth: health,
+        });
+        entries.splice(0, entries.length, ...audit.entries);
+        health.splice(0, health.length, ...audit.health);
+      } catch (error) {
+        failure = error;
+        return false;
+      }
+    },
+  });
+  if (failure) {
+    throw failure;
+  }
+  if (report.aborted) {
+    return { report, audit: null };
+  }
+  const audit = await clean(report, {
+    ...options,
+    priorEntries: entries,
+    priorHealth: health,
+  });
+  return { report, audit };
+}