disk-space-saviour 0.15.2 → 0.17.0

package/src/env/local.js

@@ -1,3 +1,5 @@
+import { checkSignal } from './scope.js';
+import { retryProcessOwners, inspectionHint } from './inspection.js';
 /**
  * Base names of a process's executable and `argv[0]`, without duplicates
  * and without the ` (deleted)` suffix of replaced binaries.
@@ -58,9 +60,13 @@ import os from 'node:os';
 import path from 'node:path';
 import { URL, fileURLToPath } from 'node:url';
 import { procIdentity, procOpenPaths } from './proc.js';
+import { CACHE_VARIABLES } from './cache-roots.js';
 
 import { hostExecutor } from '../exec.js';
 import { matchesGlob } from '../paths.js';
+import { mapLimit } from './parallel.js';
+import { directoryFingerprint, SIZE_MAX_AGE } from '../scan-state.js';
+import { indexedDirectories } from './platform-index.js';
 
 // Directories read concurrently while measuring usage.
 const WALK_BATCH = 64;
@@ -189,11 +195,11 @@ export class LocalEnv {
     this.protectedPaths = [fileURLToPath(new URL('../../', import.meta.url))];
     this.currentHome = os.homedir();
     this.skipNames = new Set(options.skipNames ?? DEFAULT_SKIP_NAMES);
-    this.vars = options.vars ?? {
-      LOCALAPPDATA: process.env.LOCALAPPDATA,
-      APPDATA: process.env.APPDATA,
-      TMPDIR: process.env.TMPDIR,
-    };
+    this.vars =
+      options.vars ??
+      Object.fromEntries(
+        CACHE_VARIABLES.map((name) => [name, process.env[name]])
+      );
   }
 
   recordAccessError(target, error) {
@@ -249,23 +255,29 @@ export class LocalEnv {
       size: stats.size,
       bytes: diskBytes(stats),
       mtimeMs: stats.mtimeMs,
+      ctimeMs: stats.ctimeMs,
       inode: `${stats.dev}:${stats.ino}`,
       uid: stats.uid,
+      gid: stats.gid,
+      mode: stats.mode,
       device: stats.dev,
     };
   }
 
   async readText(target, maxBytes = 1024 * 1024) {
+    checkSignal(this.signal);
     try {
       const handle = await fsp.open(target, 'r');
       try {
         const buffer = Buffer.alloc(maxBytes);
         const { bytesRead } = await handle.read(buffer, 0, maxBytes, 0);
+        checkSignal(this.signal);
         return buffer.subarray(0, bytesRead).toString('utf8');
       } finally {
         await handle.close();
       }
     } catch {
+      checkSignal(this.signal);
       return null;
     }
   }
@@ -287,31 +299,36 @@ export class LocalEnv {
     return heads;
   }
 
-  async list(dir) {
+  async list(dir, { discovery = false } = {}) {
     let names;
     try {
-      names = await (this.options.fileFs ?? fsp).readdir(dir);
+      names = await (this.options.fileFs ?? fsp).readdir(dir, {
+        withFileTypes: true,
+      });
     } catch (error) {
       this.recordAccessError(dir, error);
       return [];
     }
-    const entries = await Promise.all(
-      names.map(async (name) => {
-        const full = path.join(dir, name);
-        const stats = await this.#lstat(full);
-        if (!stats) {
-          return null;
-        }
-        return {
-          name,
-          path: full,
-          type: entryType(stats),
-          size: stats.size,
-          bytes: diskBytes(stats),
-          mtimeMs: stats.mtimeMs,
-        };
-      })
-    );
+    const entries = await mapLimit(names, 32, async (entry) => {
+      const name = typeof entry === 'string' ? entry : entry.name;
+      const full = path.join(dir, name);
+      if (discovery && typeof entry !== 'string' && !entry.isDirectory()) {
+        return { name, path: full, type: entryType(entry), mtimeMs: 0 };
+      }
+      const stats = await this.#lstat(full);
+      if (!stats) {
+        return null;
+      }
+      return {
+        name,
+        path: full,
+        type: entryType(stats),
+        size: stats.size,
+        bytes: diskBytes(stats),
+        mtimeMs: stats.mtimeMs,
+        device: stats.dev,
+      };
+    });
     return entries.filter(Boolean);
   }
 
@@ -320,14 +337,15 @@ export class LocalEnv {
    * counted a single time, symlinks not followed. Also reports the newest
    * file mtime.
    */
-  async usage(target) {
-    return (await this.usageMany([target])).get(target) ?? null;
+  async usage(target, options) {
+    return (await this.usageMany([target], options)).get(target) ?? null;
   }
 
   async statMany(targets) {
-    const entries = await Promise.all(
-      targets.map(async (target) => [target, await this.stat(target)])
-    );
+    const entries = await mapLimit(targets, 32, async (target) => [
+      target,
+      await this.stat(target),
+    ]);
     return new Map(entries.filter(([, stat]) => stat));
   }
 
@@ -354,6 +372,9 @@ export class LocalEnv {
       apparentBytes += stats.size;
       if (!stats.isDirectory()) {
         files++;
+        if (this.scanMetrics) {
+          this.scanMetrics.filesSized++;
+        }
         newestMtimeMs = Math.max(newestMtimeMs, stats.mtimeMs);
       }
     };
@@ -367,12 +388,10 @@ export class LocalEnv {
         errors++;
         return [];
       }
-      const children = await Promise.all(
-        names.map(async (name) => {
-          const full = path.join(dir, name);
-          return [full, await this.#lstat(full)];
-        })
-      );
+      const children = await mapLimit(names, 32, async (name) => {
+        const full = path.join(dir, name);
+        return [full, await this.#lstat(full)];
+      });
       const subdirs = [];
       for (const [full, stats] of children) {
         if (!stats) {
@@ -394,6 +413,7 @@ export class LocalEnv {
     };
     let frontier = root.isDirectory() ? [target] : [];
     while (frontier.length > 0) {
+      checkSignal(this.signal);
       const next = [];
       for (let index = 0; index < frontier.length; index += WALK_BATCH) {
         const found = await Promise.all(
@@ -444,6 +464,10 @@ export class LocalEnv {
     return results;
   }
 
+  indexedDirs(roots, globs, maxDepth) {
+    return indexedDirectories(this, roots, globs, maxDepth);
+  }
+
   /**
    * Finds regular files whose basename matches one of `names`.
    * @returns {Promise<string[]>}
@@ -459,7 +483,14 @@ export class LocalEnv {
           results.push(entry.path);
         }
       }
-      return null;
+      return new Set([
+        'node_modules',
+        'target',
+        'vendor',
+        '.venv',
+        'venv',
+        ...this.skipNames,
+      ]);
     });
     return results;
   }
@@ -471,54 +502,214 @@ export class LocalEnv {
   async #walk(roots, maxDepth, skipNames, visit) {
     const skip = skipNames ? new Set(skipNames) : this.skipNames;
     const visited = new Set();
-    const queue = await Promise.all(
+    let queue = await Promise.all(
       roots.map(async (root) => [root, 0, await this.deviceId(root)])
     );
     while (queue.length > 0) {
-      const [dir, depth, device] = queue.shift();
-      if (visited.has(dir)) {
-        continue;
-      }
-      visited.add(dir);
-      const entries = [];
-      for (const entry of await this.list(dir)) {
-        if (
-          entry.type !== 'dir' ||
-          (await this.deviceId(entry.path)) === device
-        ) {
-          entries.push(entry);
+      checkSignal(this.signal);
+      const current = queue;
+      queue = [];
+      const children = await mapLimit(
+        current,
+        8,
+        async ([dir, depth, device]) => {
+          if (visited.has(dir)) {
+            return [];
+          }
+          visited.add(dir);
+          const entries = [];
+          for (const entry of await this.discoveryList(dir)) {
+            if (
+              entry.type !== 'dir' ||
+              (entry.device ?? (await this.deviceId(entry.path))) === device
+            ) {
+              entries.push(entry);
+            }
+          }
+          const pruned = visit(dir, entries) ?? new Set();
+          if (depth + 1 >= maxDepth) {
+            return [];
+          }
+          const next = [];
+          for (const entry of entries) {
+            if (
+              entry.type === 'dir' &&
+              !pruned.has(entry.name) &&
+              !this.skipDiscoveryDirectory(dir, entry.name, skip) &&
+              (entry.device ?? (await this.deviceId(entry.path))) === device
+            ) {
+              next.push([entry.path, depth + 1, device]);
+            }
+          }
+          return next;
         }
+      );
+      queue = children.flat();
+    }
+  }
+
+  skipDiscoveryDirectory(dir, name, skip = this.skipNames) {
+    if (skip.has(name) || /\.photoslibrary$/i.test(name)) {
+      return true;
+    }
+    const homes = this.options.homes ?? [this.currentHome];
+    return (
+      (['Music', 'Pictures', 'Movies'].includes(name) && homes.includes(dir)) ||
+      (name === 'Volumes' && dir === this.path.parse(dir).root)
+    );
+  }
+
+  async discoveryStat(dir) {
+    let stats = await this.stat(dir);
+    // Explicit roots may be aliases (for example /tmp on macOS). Child links
+    // are never queued by the walker, so only an explicitly supplied root follows.
+    if (stats?.type === 'symlink') {
+      const resolved = await this.realPath(dir);
+      stats = resolved ? await this.stat(resolved) : null;
+    }
+    return stats;
+  }
+
+  async discoveryList(dir) {
+    checkSignal(this.signal);
+    const stats = await this.discoveryStat(dir);
+    if (!stats || stats.type !== 'dir') {
+      return [];
+    }
+    const key = `list:${dir}`;
+    const fingerprint = `${directoryFingerprint(stats)}:${JSON.stringify([this.scanPolicy?.exclude, this.scanPolicy?.pruned])}`;
+    let entries = this.scanIndex?.get(key, fingerprint);
+    if (entries) {
+      this.scanMetrics.indexHits++;
+    } else {
+      const errors = this.accessErrors?.size ?? 0;
+      entries = await this.list(dir, { discovery: true });
+      if ((this.accessErrors?.size ?? 0) === errors) {
+        this.scanIndex?.set(key, fingerprint, entries);
       }
-      const pruned = visit(dir, entries) ?? new Set();
-      if (depth + 1 >= maxDepth) {
-        continue;
-      }
-      for (const entry of entries) {
-        if (
-          entry.type === 'dir' &&
-          !pruned.has(entry.name) &&
-          !skip.has(entry.name) &&
-          (await this.deviceId(entry.path)) === device
-        ) {
-          queue.push([entry.path, depth + 1, device]);
+    }
+    // Stored metadata is a discovery hint; never accept external paths from it.
+    const listing = entries
+      .filter(
+        (entry) => entry.name && this.path.basename(entry.name) === entry.name
+      )
+      .map((entry) => ({ ...entry, path: this.path.join(dir, entry.name) }))
+      .filter(
+        (entry) => !this.scanPolicy || this.scanPolicy.allows(entry.path)
+      );
+    // Mounts and link replacements need not change the parent's mtime.
+    return (
+      await mapLimit(listing, 32, async (entry) => {
+        if (entry.type !== 'dir') {
+          return entry;
         }
-      }
+        const fresh = await this.stat(entry.path);
+        return fresh ? { ...entry, ...fresh } : null;
+      })
+    ).filter(Boolean);
+  }
+
+  /** Source activity excludes generated trees, and stops as soon as recent work is seen. */
+  async projectActivity(project, globs, since) {
+    let newest = 0;
+    let frontier = [project];
+    const device = await this.deviceId(project);
+    while (frontier.length && newest < since) {
+      const children = await mapLimit(frontier, 8, async (dir) => {
+        checkSignal(this.signal);
+        const next = [];
+        for (const entry of await this.list(dir)) {
+          if (entry.type === 'dir') {
+            if (
+              !this.skipDiscoveryDirectory(dir, entry.name) &&
+              !globs.some((glob) => matchesGlob(entry.name, glob)) &&
+              entry.device === device
+            ) {
+              next.push(entry.path);
+            }
+          } else if (entry.type === 'file') {
+            newest = Math.max(newest, entry.mtimeMs);
+          }
+        }
+        return next;
+      });
+      frontier = children.flat();
     }
+    return newest;
   }
 
-  async usageMany(targets) {
-    const usages = await this.rawUsageMany(targets);
+  async usageMany(targets, options) {
+    const usages = await this.rawUsageMany(targets, options);
     return this.writableLayer
       ? this.writableLayer.measure(this, usages)
       : usages;
   }
 
-  async rawUsageMany(targets) {
-    const usages = new Map();
-    for (const target of targets) {
-      usages.set(target, await this.rawUsage(target));
+  async rawUsageMany(targets, { mtime = true, cache = false } = {}) {
+    const entries = await mapLimit([...new Set(targets)], 4, async (target) => [
+      target,
+      !this.writableLayer && cache
+        ? await this.cachedUsage(target, mtime)
+        : !mtime && !this.writableLayer
+          ? await this.quickUsage(target)
+          : await this.rawUsage(target),
+    ]);
+    return new Map(entries);
+  }
+
+  async cachedUsage(target, mtime) {
+    const stats = await this.stat(target);
+    if (!stats) {
+      return null;
+    }
+    const key = `usage:${mtime}:${target}`;
+    const fingerprint = `${directoryFingerprint(stats)}:${JSON.stringify([this.scanPolicy?.exclude, this.scanPolicy?.pruned])}`;
+    const cached = this.scanIndex?.get(key, fingerprint, SIZE_MAX_AGE);
+    if (cached) {
+      this.scanMetrics.sizeCacheHits++;
+      return { ...cached, estimated: true };
     }
-    return usages;
+    const usage = mtime
+      ? await this.rawUsage(target)
+      : await this.quickUsage(target);
+    if (usage && !usage.errors) {
+      this.scanIndex?.set(key, fingerprint, usage);
+    }
+    return usage;
+  }
+
+  async quickUsage(target) {
+    const stats = await this.stat(target);
+    if (!stats) {
+      return null;
+    }
+    // du cannot enforce path-glob exclusions or detect a nested mount beneath a candidate.
+    if (
+      this.platform === 'win32' ||
+      this.scanPolicy?.exclude.length ||
+      this.scanPolicy?.mounts.some(
+        (mount) =>
+          mount.path !== '/' && mount.path.startsWith(`${target}${path.sep}`)
+      )
+    ) {
+      return this.rawUsage(target);
+    }
+    const result = await this.run(['du', '-skx', target]);
+    const match = /^\s*(\d+)\s/.exec(result.stdout);
+    if (result.code !== 0 || !match) {
+      return this.rawUsage(target);
+    }
+    const usage = {
+      bytes: Number(match[1]) * 1024,
+      apparentBytes: Number(match[1]) * 1024,
+      newestMtimeMs: 0,
+      files: 0,
+      errors: 0,
+    };
+    if (this.scanMetrics) {
+      this.scanMetrics.directoriesSized++;
+    }
+    return usage;
   }
 
   async readLink(target) {
@@ -613,11 +804,15 @@ export class LocalEnv {
 
   async #linuxProcesses() {
     const io = this.options.procFs ?? fsp;
+    this.mountNamespace = await io
+      .readlink('/proc/self/ns/mnt')
+      .catch(() => null);
     const pids = (await io.readdir('/proc')).filter((name) =>
       /^\d+$/.test(name)
     );
     const processes = [];
     for (const pid of pids) {
+      checkSignal(this.signal);
       const base = `/proc/${pid}`;
       const stat = parseProcStat(
         await io.readFile(`${base}/stat`, 'utf8').catch(() => '')
@@ -641,6 +836,7 @@ export class LocalEnv {
         ...(await procIdentity(base, io)),
         name,
         startTime: stat.startTime,
+        mountNamespace: await io.readlink(`${base}/ns/mnt`).catch(() => null),
         cwd,
         aliases: processAliases(exe, cmdline.split('\0')[0]),
         command: cmdline.replaceAll('\0', ' '),
@@ -658,29 +854,8 @@ export class LocalEnv {
   async openPaths({ strict = true } = {}) {
     try {
       let paths = await this.rawOpenPaths({ strict });
-      if (strict && this.unreadableProcesses?.length && this.processInspector) {
-        const privileged = await this.processInspector.openPaths();
-        if (privileged) {
-          this.unreadableProcesses =
-            this.processInspector.unreadableProcesses ?? [];
-          paths = privileged;
-        }
-      }
-      if (strict && this.unreadableProcesses?.length) {
-        const { OPEN_PATHS_SCRIPT } = await import('./shell.js');
-        const elevated = await this.run([
-          'sudo',
-          '-n',
-          'sh',
-          '-c',
-          OPEN_PATHS_SCRIPT,
-        ]);
-        if (elevated.code === 0) {
-          paths = new Set(
-            elevated.stdout.split('\n').filter((line) => line.startsWith('/'))
-          );
-          this.unreadableProcesses = [];
-        }
+      if (strict) {
+        paths = await this.#completeInspection(paths);
       }
       this.#processHint();
       return paths;
@@ -688,6 +863,9 @@ export class LocalEnv {
       if (this.processInspector) {
         const paths = await this.processInspector.openPaths();
         this.probeDiagnostic = this.processInspector.probeDiagnostic;
+        this.unreadableProcesses =
+          this.processInspector.unreadableProcesses ?? [];
+        this.#processHint();
         return paths;
       }
       this.probeDiagnostic = error.message;
@@ -695,16 +873,56 @@ export class LocalEnv {
     }
   }
 
+  async #completeInspection(paths) {
+    paths = await this.#inspectOwners(paths);
+    if (this.unreadableProcesses?.length && this.processInspector) {
+      const privileged = await this.processInspector.openPaths();
+      if (privileged) {
+        this.unreadableProcesses =
+          this.processInspector.unreadableProcesses ?? [];
+        paths = new Set([...(paths ?? []), ...privileged]);
+      }
+    }
+    if (this.unreadableProcesses?.length) {
+      const { OPEN_PATHS_SCRIPT } = await import('./shell.js');
+      const elevated = await this.run([
+        'sudo',
+        '-n',
+        'sh',
+        '-c',
+        OPEN_PATHS_SCRIPT,
+      ]);
+      if (elevated.code === 0) {
+        paths = new Set(
+          elevated.stdout.split('\n').filter((line) => line.startsWith('/'))
+        );
+        this.unreadableProcesses = [];
+      }
+    }
+    return paths;
+  }
+
   #processHint() {
-    this.probeHint = this.unreadableProcesses?.length
-      ? `${this.unreadableProcesses.length} processes are unreadable; uncertainty is scoped to their paths and UID. Re-run as root for complete inspection (docker exec -u 0 -e HOME=<user-home> <container> dss scan).`
-      : null;
+    this.probeHint = inspectionHint(this.unreadableProcesses ?? []);
+  }
+
+  async #inspectOwners(paths) {
+    if (!this.unreadableProcesses?.length || !(await this.isRoot())) {
+      return paths;
+    }
+    const evidence = await retryProcessOwners(this.executor, {
+      paths,
+      unreadable: this.unreadableProcesses,
+    });
+    this.unreadableProcesses = evidence.unreadable;
+    return evidence.paths;
   }
 
   async rawOpenPaths({ strict = true } = {}) {
     if (this.platform === 'linux') {
       const { paths, unreadable } = await procOpenPaths(
-        this.options.procFs ?? fsp
+        this.options.procFs ?? fsp,
+        this.signal
       );
       this.unreadableProcesses = strict ? unreadable : [];
       return paths;

package/src/env/parallel.js

@@ -0,0 +1,23 @@
+/** Finite worker pool; settle all in-flight work before propagating failure. */
+export async function mapLimit(values, limit, operation) {
+  const results = new Array(values.length);
+  let cursor = 0;
+  let failure;
+  const worker = async () => {
+    while (!failure && cursor < values.length) {
+      const index = cursor++;
+      try {
+        results[index] = await operation(values[index], index);
+      } catch (error) {
+        failure ??= error;
+      }
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, values.length) }, worker)
+  );
+  if (failure) {
+    throw failure;
+  }
+  return results;
+}

package/src/env/platform-index.js

@@ -0,0 +1,87 @@
+/** OS index hits are hints; stale/missing indexes always fall back to discovery. */
+import { isWithin, matchesGlob } from '../paths.js';
+import { mapLimit } from './parallel.js';
+
+export async function indexedDirectories(env, roots, globs, maxDepth) {
+  if (!env.scanIndex?.directory || env.platformIndex === false) {
+    return [];
+  }
+  let hits = [];
+  try {
+    if (env.platform === 'darwin' && (await env.which('mdfind'))) {
+      const query = globs
+        .map((glob) => `kMDItemFSName == "${glob.replaceAll('"', '\\"')}"`)
+        .join(' || ');
+      for (const root of roots) {
+        const result = await env.run(['mdfind', '-0', '-onlyin', root, query]);
+        if (result.code === 0) {
+          hits.push(...result.stdout.split('\0').filter(Boolean));
+        }
+      }
+    } else if (env.platform === 'linux') {
+      const command = (await env.which('plocate'))
+        ? 'plocate'
+        : (await env.which('locate'))
+          ? 'locate'
+          : null;
+      if (command) {
+        const result = await env.run([command, '-0', '-e', '-b', ...globs]);
+        if (result.code === 0) {
+          hits = result.stdout.split('\0').filter(Boolean);
+        }
+      }
+    }
+  } catch {
+    // Budget/cancellation will be observed by the fallback walk too.
+  }
+  hits = [...new Set(hits)]
+    .filter((target) => {
+      const root = roots.find((candidate) =>
+        isWithin(target, candidate, env.path)
+      );
+      if (
+        !root ||
+        !env.path.isAbsolute(target) ||
+        !globs.some((glob) => matchesGlob(env.path.basename(target), glob))
+      ) {
+        return false;
+      }
+      const parts = env.path.relative(root, target).split(/[\\/]/);
+      return (
+        parts.length <= maxDepth &&
+        !parts
+          .slice(0, -1)
+          .some(
+            (name, index) =>
+              env.skipDiscoveryDirectory(
+                env.path.join(root, ...parts.slice(0, index)),
+                name
+              ) || globs.some((glob) => matchesGlob(name, glob))
+          )
+      );
+    })
+    .slice(0, 20000);
+  const candidates = await mapLimit(hits, 8, async (target) => {
+    const stat = await env.stat(target);
+    const root = roots.find((candidate) =>
+      isWithin(target, candidate, env.path)
+    );
+    if (stat?.type !== 'dir' || stat.device !== (await env.deviceId(root))) {
+      return null;
+    }
+    let parent = env.path.dirname(target);
+    while (parent !== root && isWithin(parent, root, env.path)) {
+      const ancestor = await env.stat(parent);
+      if (ancestor?.type !== 'dir' || ancestor.device !== stat.device) {
+        return null;
+      }
+      parent = env.path.dirname(parent);
+    }
+    return {
+      path: target,
+      name: env.path.basename(target),
+      parent: env.path.dirname(target),
+    };
+  });
+  return candidates.filter(Boolean);
+}