disk-space-saviour 0.13.0 → 0.15.1

package/src/env/local.js

@@ -56,6 +56,8 @@ export function parseLsofCwds(output) {
 import { promises as fsp } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { URL, fileURLToPath } from 'node:url';
+import { procIdentity, procOpenPaths } from './proc.js';
 
 import { hostExecutor } from '../exec.js';
 import { matchesGlob } from '../paths.js';
@@ -184,6 +186,7 @@ export class LocalEnv {
     this.platform = options.platform ?? process.platform;
     this.executor = options.executor ?? hostExecutor();
     this.options = options;
+    this.protectedPaths = [fileURLToPath(new URL('../../', import.meta.url))];
     this.currentHome = os.homedir();
     this.skipNames = new Set(options.skipNames ?? DEFAULT_SKIP_NAMES);
     this.vars = options.vars ?? {
@@ -193,6 +196,30 @@ export class LocalEnv {
     };
   }
 
+  recordAccessError(target, error) {
+    if (!['EACCES', 'EPERM'].includes(error.code)) {
+      return;
+    }
+    this.accessErrors ??= new Map();
+    const advice =
+      this.platform === 'darwin' && /Safari|WebKit/.test(target)
+        ? '; grant Full Disk Access to the terminal running dss'
+        : '';
+    this.accessErrors.set(
+      target,
+      `cannot inspect ${target}: ${error.code}${advice}`
+    );
+  }
+
+  async #lstat(target) {
+    try {
+      return await (this.options.fileFs ?? fsp).lstat(target);
+    } catch (error) {
+      this.recordAccessError(target, error);
+      return null;
+    }
+  }
+
   homeDirs() {
     return Promise.resolve(this.options.homes ?? [os.homedir()]);
   }
@@ -209,11 +236,11 @@ export class LocalEnv {
   }
 
   async exists(target) {
-    return (await safeLstat(target)) !== null;
+    return (await this.#lstat(target)) !== null;
   }
 
   async stat(target) {
-    const stats = await safeLstat(target);
+    const stats = await this.#lstat(target);
     if (!stats) {
       return null;
     }
@@ -222,6 +249,9 @@ export class LocalEnv {
       size: stats.size,
       bytes: diskBytes(stats),
       mtimeMs: stats.mtimeMs,
+      inode: `${stats.dev}:${stats.ino}`,
+      uid: stats.uid,
+      device: stats.dev,
     };
   }
 
@@ -260,14 +290,15 @@ export class LocalEnv {
   async list(dir) {
     let names;
     try {
-      names = await fsp.readdir(dir);
-    } catch {
+      names = await (this.options.fileFs ?? fsp).readdir(dir);
+    } catch (error) {
+      this.recordAccessError(dir, error);
       return [];
     }
     const entries = await Promise.all(
       names.map(async (name) => {
         const full = path.join(dir, name);
-        const stats = await safeLstat(full);
+        const stats = await this.#lstat(full);
         if (!stats) {
           return null;
         }
@@ -290,7 +321,18 @@ export class LocalEnv {
    * file mtime.
    */
   async usage(target) {
-    const root = await safeLstat(target);
+    return (await this.usageMany([target])).get(target) ?? null;
+  }
+
+  async statMany(targets) {
+    const entries = await Promise.all(
+      targets.map(async (target) => [target, await this.stat(target)])
+    );
+    return new Map(entries.filter(([, stat]) => stat));
+  }
+
+  async rawUsage(target) {
+    const root = await this.#lstat(target);
     if (!root) {
       return null;
     }
@@ -319,15 +361,16 @@ export class LocalEnv {
     const visit = async (dir) => {
       let names;
       try {
-        names = await fsp.readdir(dir);
-      } catch {
+        names = await (this.options.fileFs ?? fsp).readdir(dir);
+      } catch (error) {
+        this.recordAccessError(dir, error);
         errors++;
         return [];
       }
       const children = await Promise.all(
         names.map(async (name) => {
           const full = path.join(dir, name);
-          return [full, await safeLstat(full)];
+          return [full, await this.#lstat(full)];
         })
       );
       const subdirs = [];
@@ -336,6 +379,12 @@ export class LocalEnv {
           errors++;
           continue;
         }
+        if (
+          stats.dev !== root.dev ||
+          (this.scanPolicy && !this.scanPolicy.allows(full))
+        ) {
+          continue;
+        }
         account(stats);
         if (stats.isDirectory()) {
           subdirs.push(full);
@@ -422,14 +471,24 @@ export class LocalEnv {
   async #walk(roots, maxDepth, skipNames, visit) {
     const skip = skipNames ? new Set(skipNames) : this.skipNames;
     const visited = new Set();
-    const queue = roots.map((root) => [root, 0]);
+    const queue = await Promise.all(
+      roots.map(async (root) => [root, 0, await this.deviceId(root)])
+    );
     while (queue.length > 0) {
-      const [dir, depth] = queue.shift();
+      const [dir, depth, device] = queue.shift();
       if (visited.has(dir)) {
         continue;
       }
       visited.add(dir);
-      const entries = await this.list(dir);
+      const entries = [];
+      for (const entry of await this.list(dir)) {
+        if (
+          entry.type !== 'dir' ||
+          (await this.deviceId(entry.path)) === device
+        ) {
+          entries.push(entry);
+        }
+      }
       const pruned = visit(dir, entries) ?? new Set();
       if (depth + 1 >= maxDepth) {
         continue;
@@ -438,18 +497,26 @@ export class LocalEnv {
         if (
           entry.type === 'dir' &&
           !pruned.has(entry.name) &&
-          !skip.has(entry.name)
+          !skip.has(entry.name) &&
+          (await this.deviceId(entry.path)) === device
         ) {
-          queue.push([entry.path, depth + 1]);
+          queue.push([entry.path, depth + 1, device]);
         }
       }
     }
   }
 
   async usageMany(targets) {
+    const usages = await this.rawUsageMany(targets);
+    return this.writableLayer
+      ? this.writableLayer.measure(this, usages)
+      : usages;
+  }
+
+  async rawUsageMany(targets) {
     const usages = new Map();
     for (const target of targets) {
-      usages.set(target, await this.usage(target));
+      usages.set(target, await this.rawUsage(target));
     }
     return usages;
   }
@@ -492,6 +559,13 @@ export class LocalEnv {
 
   async processes() {
     const selfPids = new Set([process.pid, process.ppid]);
+    if (this.processInspector?.probeExecutor) {
+      return (
+        (await this.processInspector.processes())?.filter(
+          (proc) => !selfPids.has(proc.pid)
+        ) ?? null
+      );
+    }
     if (this.platform === 'linux') {
       return (await this.#linuxProcesses()).filter(
         (proc) => !selfPids.has(proc.pid)
@@ -507,6 +581,13 @@ export class LocalEnv {
         .filter((proc) => !selfPids.has(proc.pid));
     }
     const result = await this.run(['ps', '-axo', 'pid=,comm=']);
+    const commands = new Map(
+      (await this.run(['ps', '-axo', 'pid=,args='])).stdout
+        .split('\n')
+        .map((line) => /^\s*(\d+)\s+(.+)$/.exec(line))
+        .filter(Boolean)
+        .map((match) => [Number(match[1]), match[2]])
+    );
     const cwds =
       this.platform === 'darwin'
         ? parseLsofCwds(
@@ -525,44 +606,45 @@ export class LocalEnv {
         pid: Number(m[1]),
         name: path.basename(m[2].trim()),
         cwd: cwds.get(Number(m[1])) ?? null,
+        command: commands.get(Number(m[1])) ?? '',
       }))
       .filter((proc) => !selfPids.has(proc.pid));
   }
 
   async #linuxProcesses() {
-    let pids = [];
-    try {
-      pids = (await fsp.readdir('/proc')).filter((name) => /^\d+$/.test(name));
-    } catch {
-      return [];
-    }
+    const io = this.options.procFs ?? fsp;
+    const pids = (await io.readdir('/proc')).filter((name) =>
+      /^\d+$/.test(name)
+    );
     const processes = [];
     for (const pid of pids) {
       const base = `/proc/${pid}`;
+      const stat = parseProcStat(
+        await io.readFile(`${base}/stat`, 'utf8').catch(() => '')
+      );
+      if (stat.state === 'Z') {
+        continue;
+      }
       let name;
       try {
-        name = (await fsp.readFile(`${base}/comm`, 'utf8')).trim();
+        name = (await io.readFile(`${base}/comm`, 'utf8')).trim();
       } catch {
         continue;
       }
-      const cwd = await fsp.readlink(`${base}/cwd`).catch(() => null);
-      const exe = await fsp.readlink(`${base}/exe`).catch(() => '');
-      const cmdline = await fsp
+      const cwd = await io.readlink(`${base}/cwd`).catch(() => null);
+      const exe = await io.readlink(`${base}/exe`).catch(() => '');
+      const cmdline = await io
         .readFile(`${base}/cmdline`, 'utf8')
         .catch(() => '');
-      const stat = parseProcStat(
-        await fsp.readFile(`${base}/stat`, 'utf8').catch(() => '')
-      );
-      if (stat.state === 'Z') {
-        continue;
-      }
       processes.push({
         pid: Number(pid),
+        ...(await procIdentity(base, io)),
         name,
         startTime: stat.startTime,
         cwd,
         aliases: processAliases(exe, cmdline.split('\0')[0]),
         command: cmdline.replaceAll('\0', ' '),
+        argv: cmdline.split('\0').filter(Boolean),
       });
     }
     return processes;
@@ -574,55 +656,57 @@ export class LocalEnv {
    * @returns {Promise<Set<string>|null>}
    */
   async openPaths({ strict = true } = {}) {
-    if (this.platform === 'linux') {
-      const paths = new Set();
-      let pids = [];
-      try {
-        pids = (await fsp.readdir('/proc')).filter((n) => /^\d+$/.test(n));
-      } catch {
-        return null;
-      }
-      for (const pid of pids) {
-        const base = `/proc/${pid}`;
-        for (const link of ['cwd', 'exe']) {
-          const target = await fsp
-            .readlink(`${base}/${link}`)
-            .catch((error) => {
-              if (
-                strict &&
-                (error.code === 'EACCES' || error.code === 'EPERM')
-              ) {
-                throw error;
-              }
-              return '';
-            });
-          if (target.startsWith('/')) {
-            paths.add(target.replace(/ \(deleted\)$/, ''));
-          }
+    try {
+      let paths = await this.rawOpenPaths({ strict });
+      if (strict && this.unreadableProcesses?.length && this.processInspector) {
+        const privileged = await this.processInspector.openPaths();
+        if (privileged) {
+          this.unreadableProcesses =
+            this.processInspector.unreadableProcesses ?? [];
+          paths = privileged;
         }
-        const fds = await fsp.readdir(`${base}/fd`).catch((error) => {
-          if (strict && (error.code === 'EACCES' || error.code === 'EPERM')) {
-            throw error;
-          }
-          return [];
-        });
-        for (const fd of fds) {
-          const target = await fsp
-            .readlink(`${base}/fd/${fd}`)
-            .catch((error) => {
-              if (
-                strict &&
-                (error.code === 'EACCES' || error.code === 'EPERM')
-              ) {
-                throw error;
-              }
-              return '';
-            });
-          if (target.startsWith('/')) {
-            paths.add(target.replace(/ \(deleted\)$/, ''));
-          }
+      }
+      if (strict && this.unreadableProcesses?.length) {
+        const { OPEN_PATHS_SCRIPT } = await import('./shell.js');
+        const elevated = await this.run([
+          'sudo',
+          '-n',
+          'sh',
+          '-c',
+          OPEN_PATHS_SCRIPT,
+        ]);
+        if (elevated.code === 0) {
+          paths = new Set(
+            elevated.stdout.split('\n').filter((line) => line.startsWith('/'))
+          );
+          this.unreadableProcesses = [];
         }
       }
+      this.#processHint();
+      return paths;
+    } catch (error) {
+      if (this.processInspector) {
+        const paths = await this.processInspector.openPaths();
+        this.probeDiagnostic = this.processInspector.probeDiagnostic;
+        return paths;
+      }
+      this.probeDiagnostic = error.message;
+      throw error;
+    }
+  }
+
+  #processHint() {
+    this.probeHint = this.unreadableProcesses?.length
+      ? `${this.unreadableProcesses.length} processes are unreadable; uncertainty is scoped to their paths and UID. Re-run as root for complete inspection (docker exec -u 0 -e HOME=<user-home> <container> dss scan).`
+      : null;
+  }
+
+  async rawOpenPaths({ strict = true } = {}) {
+    if (this.platform === 'linux') {
+      const { paths, unreadable } = await procOpenPaths(
+        this.options.procFs ?? fsp
+      );
+      this.unreadableProcesses = strict ? unreadable : [];
       return paths;
     }
     if (this.platform === 'darwin') {
@@ -654,7 +738,7 @@ export class LocalEnv {
         : [''];
     for (const dir of dirs.filter(Boolean)) {
       for (const extension of extensions) {
-        const stats = await safeLstat(path.join(dir, command + extension));
+        const stats = await this.#lstat(path.join(dir, command + extension));
         if (stats && !stats.isDirectory()) {
           return true;
         }

package/src/env/policy.js

@@ -0,0 +1,155 @@
+/** Shared traversal and deletion boundaries for every scanner/environment. */
+import { isWithin, matchesExcluded } from '../paths.js';
+import { trace } from '../exec.js';
+
+const RUNTIME_ROOTS = [
+  '/var/lib/docker',
+  '/var/lib/containerd',
+  '/run/containerd',
+  '/var/lib/containers',
+];
+const unescapeMount = (value) =>
+  value.replace(/\\([0-7]{3})/g, (_, octal) =>
+    String.fromCharCode(parseInt(octal, 8))
+  );
+
+export function parseMountInfo(text) {
+  return (text ?? '').split('\n').flatMap((line) => {
+    const [left, right] = line.split(' - ');
+    if (!right) {
+      return [];
+    }
+    const fields = left.split(' ');
+    const [type, source, options = ''] = right.split(' ');
+    return [
+      {
+        path: unescapeMount(fields[4]),
+        type,
+        source,
+        readonly: fields[5].split(',').includes('ro'),
+        options,
+      },
+    ];
+  });
+}
+
+export class ScanPolicy {
+  constructor(env, options, mounts) {
+    this.env = env;
+    this.exclude = options.exclude ?? [];
+    this.roots = options.roots ?? [];
+    this.mounts = mounts;
+    this.excludedPaths = new Set();
+    this.pruned = mounts
+      .filter(
+        (mount) =>
+          mount.path !== '/' &&
+          (mount.readonly ||
+            mount.type === 'overlay' ||
+            mount.type === 'fuse.overlayfs' ||
+            !this.roots.some((root) => isWithin(root, mount.path, env.path)))
+      )
+      .map((mount) => mount.path);
+    this.overlay = mounts.find(
+      (mount) =>
+        mount.path === '/' && ['overlay', 'fuse.overlayfs'].includes(mount.type)
+    );
+  }
+
+  allows(target) {
+    const { path } = this.env;
+    if (matchesExcluded(target, this.exclude, path)) {
+      this.excludedPaths.add(target);
+      return false;
+    }
+    return (
+      !RUNTIME_ROOTS.some((root) => isWithin(target, root, path)) &&
+      !matchesExcluded(
+        target,
+        ['/tmp/containerd-mount*', '/private/tmp/containerd-mount*'],
+        path
+      ) &&
+      !this.pruned.some((root) => isWithin(target, root, path)) &&
+      !this.mounts.some(
+        (mount) => mount.readonly && isWithin(target, mount.path, path)
+      )
+    );
+  }
+
+  removalReason(target) {
+    if (!this.allows(target)) {
+      return 'path is excluded, mounted, read-only or owned by a container runtime';
+    }
+    if (
+      [...this.excludedPaths].some((root) =>
+        isWithin(root, target, this.env.path)
+      ) ||
+      this.exclude.some(
+        (pattern) =>
+          !/[?*[]/.test(pattern) && isWithin(pattern, target, this.env.path)
+      )
+    ) {
+      return 'contains an excluded path';
+    }
+    if (this.pruned.some((root) => isWithin(root, target, this.env.path))) {
+      return 'contains a mounted filesystem';
+    }
+    return null;
+  }
+}
+
+/** Install boundaries before any scanner expands paths or traverses roots. */
+export async function configureScanPolicy(env, options) {
+  const mounts =
+    env.platform === 'linux' && env.readText
+      ? parseMountInfo(await env.readText('/proc/self/mountinfo'))
+      : [];
+  const excluded = env.scanPolicy?.excludedPaths;
+  env.scanPolicy = new ScanPolicy(env, options, mounts);
+  if (excluded) {
+    env.scanPolicy.excludedPaths = excluded;
+  }
+  if (env.policyInstalled) {
+    return;
+  }
+  env.policyInstalled = true;
+  const allowed = (target) => env.scanPolicy.allows(target);
+  for (const name of ['list', 'stat', 'exists']) {
+    if (!env[name]) {
+      continue;
+    }
+    const original = env[name].bind(env);
+    env[name] = async (target, ...args) => {
+      if (!allowed(target)) {
+        trace('pruned', target);
+        return name === 'list' ? [] : name === 'exists' ? false : null;
+      }
+      const value = await original(target, ...args);
+      return name === 'list'
+        ? value.filter((entry) =>
+            allowed(entry.path ?? env.path.join(target, entry.name))
+          )
+        : value;
+    };
+  }
+  for (const name of ['listMany', 'existsMany', 'statMany', 'rawUsageMany']) {
+    if (!env[name]) {
+      continue;
+    }
+    const original = env[name].bind(env);
+    env[name] = async (targets, ...args) => {
+      const value = await original(targets.filter(allowed), ...args);
+      if (name !== 'listMany') {
+        return value;
+      }
+      return new Map(
+        [...value].map(([dir, entries]) => [
+          dir,
+          entries.filter((entry) =>
+            allowed(entry.path ?? env.path.join(dir, entry.name))
+          ),
+        ])
+      );
+    };
+  }
+}

package/src/env/proc.js

@@ -0,0 +1,53 @@
+/** Linux proc inspection preserves evidence when only some PIDs are denied. */
+import { promises as fsp } from 'node:fs';
+
+export async function procIdentity(base, io = fsp) {
+  const read = (name) => io.readFile(`${base}/${name}`, 'utf8').catch(() => '');
+  const [status, comm, command] = await Promise.all([
+    read('status'),
+    read('comm'),
+    read('cmdline'),
+  ]);
+  const uid = /^Uid:\s+(\d+)/m.exec(status)?.[1];
+  return {
+    pid: Number(base.split('/').at(-1)),
+    uid: uid === undefined ? null : Number(uid),
+    name: comm.trim(),
+    argv: command.split('\0').filter(Boolean),
+    command: command.replaceAll('\0', ' '),
+  };
+}
+
+export async function procOpenPaths(io = fsp) {
+  const paths = new Set();
+  const unreadable = [];
+  const pids = (await io.readdir('/proc')).filter((pid) => /^\d+$/.test(pid));
+  for (const pid of pids) {
+    const base = `/proc/${pid}`;
+    const stat = await io.readFile(`${base}/stat`, 'utf8').catch(() => '');
+    if (stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z ')) {
+      continue;
+    }
+    let denied = false;
+    const failure = (error) => {
+      if (['EACCES', 'EPERM'].includes(error.code)) {
+        denied = true;
+      }
+      return '';
+    };
+    const fds = await io.readdir(`${base}/fd`).catch((error) => {
+      failure(error);
+      return [];
+    });
+    for (const link of ['cwd', 'exe', ...fds.map((fd) => `fd/${fd}`)]) {
+      const target = await io.readlink(`${base}/${link}`).catch(failure);
+      if (target.startsWith('/')) {
+        paths.add(target.replace(/ \(deleted\)$/, ''));
+      }
+    }
+    if (denied) {
+      unreadable.push(await procIdentity(base, io));
+    }
+  }
+  return { paths, unreadable };
+}