disk-space-saviour 0.15.2 → 0.17.0

package/src/env/proc.js

@@ -1,5 +1,6 @@
 /** Linux proc inspection preserves evidence when only some PIDs are denied. */
 import { promises as fsp } from 'node:fs';
+import { checkSignal } from './scope.js';
 
 export async function procIdentity(base, io = fsp) {
   const read = (name) => io.readFile(`${base}/${name}`, 'utf8').catch(() => '');
@@ -9,20 +10,30 @@ export async function procIdentity(base, io = fsp) {
     read('cmdline'),
   ]);
   const uid = /^Uid:\s+(\d+)/m.exec(status)?.[1];
+  const gid = /^Gid:\s+(\d+)/m.exec(status)?.[1];
   return {
     pid: Number(base.split('/').at(-1)),
     uid: uid === undefined ? null : Number(uid),
+    gid: gid === undefined ? null : Number(gid),
+    groups:
+      /^Groups:\s*(.*)$/m
+        .exec(status)?.[1]
+        .trim()
+        .split(/\s+/)
+        .filter(Boolean)
+        .map(Number) ?? null,
     name: comm.trim(),
     argv: command.split('\0').filter(Boolean),
     command: command.replaceAll('\0', ' '),
   };
 }
 
-export async function procOpenPaths(io = fsp) {
+export async function procOpenPaths(io = fsp, signal) {
   const paths = new Set();
   const unreadable = [];
   const pids = (await io.readdir('/proc')).filter((pid) => /^\d+$/.test(pid));
   for (const pid of pids) {
+    checkSignal(signal);
     const base = `/proc/${pid}`;
     const stat = await io.readFile(`${base}/stat`, 'utf8').catch(() => '');
     if (stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z ')) {
@@ -40,6 +51,7 @@ export async function procOpenPaths(io = fsp) {
       return [];
     });
     for (const link of ['cwd', 'exe', ...fds.map((fd) => `fd/${fd}`)]) {
+      checkSignal(signal);
       const target = await io.readlink(`${base}/${link}`).catch(failure);
       if (target.startsWith('/')) {
         paths.add(target.replace(/ \(deleted\)$/, ''));

package/src/env/process-scripts.js

@@ -0,0 +1,24 @@
+/** Read-only probes shared by local identity retries and Docker adapters. */
+export const OPEN_PATHS_SCRIPT = `failed=0
+[ "$#" -gt 0 ] || set -- /proc/[0-9]*
+for p do
+  [ -d "$p" ] || continue
+  state=$(sed 's/.*) //' "$p/stat" 2>/dev/null | cut -d' ' -f1)
+  [ "$state" = Z ] && continue
+  unreadable=0
+  ls "$p/fd" >/dev/null 2>&1 || unreadable=1
+  for l in "$p/cwd" "$p/exe" "$p"/fd/*; do
+    [ -L "$l" ] || continue
+    readlink "$l" 2>/dev/null || { [ -L "$l" ] && unreadable=1; }
+  done
+  if [ "$unreadable" = 1 ] && [ -d "$p" ]; then
+    printf 'unreadable pid %s (%s)\\n' "\${p#/proc/}" "$(cat "$p/comm" 2>/dev/null)" >&2
+    uid=$(awk '/^Uid:/{print $2}' "$p/status" 2>/dev/null)
+    gid=$(awk '/^Gid:/{print $2}' "$p/status" 2>/dev/null)
+    printf 'DSS_GID|%s|%s\\n' "\${p#/proc/}" "$gid" >&2
+    command=$(tr '\\000' ' ' < "$p/cmdline" 2>/dev/null)
+    printf 'DSS_UNREADABLE|%s|%s|%s|%s\\n' "\${p#/proc/}" "$uid" "$(cat "$p/comm" 2>/dev/null)" "$command" >&2
+    failed=1
+  fi
+done
+exit "$failed"`;

package/src/env/scope.js

@@ -0,0 +1,95 @@
+/** Scan cancellation and deadlines shared by local and remote adapters. */
+export function checkSignal(signal) {
+  if (signal?.aborted) {
+    const error = new Error(signal.reason?.message ?? 'scan aborted');
+    error.name = 'AbortError';
+    throw error;
+  }
+}
+
+export function signalExecutor(executor, signal, deadline = Infinity) {
+  const base = executor.unscoped ?? executor;
+  return {
+    ...executor,
+    unscoped: executor.unscoped ?? executor,
+    signal,
+    spawn: (argv, options = {}) => {
+      checkSignal(options.signal ?? signal);
+      return base.spawn(argv, {
+        ...options,
+        signal: options.signal ?? signal,
+        timeoutMs: Math.max(
+          1,
+          Math.min(options.timeoutMs ?? 600000, deadline - Date.now())
+        ),
+      });
+    },
+    run: async (argv, options = {}) => {
+      const active = options.signal ?? signal;
+      checkSignal(active);
+      const result = await base.run(argv, {
+        ...options,
+        signal: active,
+        timeoutMs: Math.max(
+          1,
+          Math.min(options.timeoutMs ?? 600000, deadline - Date.now())
+        ),
+      });
+      checkSignal(active);
+      return result;
+    },
+  };
+}
+
+/** Restore adapter state so later clean/re-scan calls never inherit an old abort. */
+export async function environmentScope(
+  env,
+  signal,
+  operation,
+  deadline = Infinity
+) {
+  const original = {
+    executor: env.executor,
+    signal: env.signal,
+    deadline: env.deadline,
+  };
+  const probe = env.probeExecutor;
+  env.signal = signal;
+  env.deadline = deadline;
+  env.executor = signalExecutor(env.executor, signal, deadline);
+  if (probe) {
+    env.probeExecutor = signalExecutor(probe, signal, deadline);
+  }
+  const layerExecutor = env.writableLayer?.docker?.executor;
+  if (layerExecutor) {
+    env.writableLayer.docker.executor = signalExecutor(
+      layerExecutor,
+      signal,
+      deadline
+    );
+  }
+  try {
+    checkSignal(signal);
+    return await (env.processInspector
+      ? environmentScope(env.processInspector, signal, operation, deadline)
+      : operation());
+  } finally {
+    Object.assign(env, original);
+    env.probeExecutor = env.probeExecutor?.unscoped ?? probe;
+    if (layerExecutor) {
+      env.writableLayer.docker.executor = layerExecutor;
+    }
+  }
+}
+
+/** Scanner arrays emit completed findings immediately, retaining them on abort. */
+export function scanItems(context) {
+  const items = [];
+  items.push = (...entries) => {
+    for (const entry of entries) {
+      context.emit?.(entry);
+    }
+    return Array.prototype.push.apply(items, entries);
+  };
+  return items;
+}

package/src/env/shell.js

@@ -1,3 +1,10 @@
+import { OPEN_PATHS_SCRIPT } from './process-scripts.js';
+import {
+  parseProcessProbe,
+  retryProcessOwners,
+  inspectionHint,
+} from './inspection.js';
+export { OPEN_PATHS_SCRIPT } from './process-scripts.js';
 /**
  * Environment adapter for a filesystem reached through an executor, usually
  * a running container via `docker exec`.
@@ -12,9 +19,10 @@ import { sharedWritableLayer } from '../docker/writable.js';
 
 import { containerExecutor, shellQuote, trace } from '../exec.js';
 import { DEFAULT_SKIP_NAMES, parseDf, processAliases } from './local.js';
+import { batches } from './batches.js';
+import { signalExecutor } from './scope.js';
 
 const STAT_FORMAT = '%F|%s|%b|%B|%Y|%n';
-const ARG_BATCH = 200;
 const RECORD = '\u0001';
 
 const LIST_SCRIPT = `for d in "$@"; do
@@ -50,27 +58,6 @@ for p in /proc/[0-9]*; do
   printf '%s|%s|%s|%s|%s|%s|%s\\n' "\${p#/proc/}" "$t" "$c" "$e" "$a" "$w" "$q"
 done`;
 
-export const OPEN_PATHS_SCRIPT = `failed=0
-for p in /proc/[0-9]*; do
-  [ -d "$p" ] || continue
-  state=$(sed 's/.*) //' "$p/stat" 2>/dev/null | cut -d' ' -f1)
-  [ "$state" = Z ] && continue
-  unreadable=0
-  ls "$p/fd" >/dev/null 2>&1 || unreadable=1
-  for l in "$p/cwd" "$p/exe" "$p"/fd/*; do
-    [ -L "$l" ] || continue
-    readlink "$l" 2>/dev/null || { [ -L "$l" ] && unreadable=1; }
-  done
-  if [ "$unreadable" = 1 ] && [ -d "$p" ]; then
-    printf 'unreadable pid %s (%s)\\n' "\${p#/proc/}" "$(cat "$p/comm" 2>/dev/null)" >&2
-    uid=$(awk '/^Uid:/{print $2}' "$p/status" 2>/dev/null)
-    command=$(tr '\\000' ' ' < "$p/cmdline" 2>/dev/null)
-    printf 'DSS_UNREADABLE|%s|%s|%s|%s\\n' "\${p#/proc/}" "$uid" "$(cat "$p/comm" 2>/dev/null)" "$command" >&2
-    failed=1
-  fi
-done
-exit "$failed"`;
-
 const REMOVE_SCRIPT = `rm -rf -- "$@" 2>/dev/null && exit 0
 chmod -R u+w -- "$@" 2>/dev/null
 rm -rf -- "$@"`;
@@ -146,14 +133,6 @@ function parseUsage(body) {
   };
 }
 
-function batches(items, size = ARG_BATCH) {
-  const result = [];
-  for (let index = 0; index < items.length; index += size) {
-    result.push(items.slice(index, index + size));
-  }
-  return result;
-}
-
 function findNameExpression(names) {
   return names.flatMap((name, index) =>
     index === 0 ? ['-name', name] : ['-o', '-name', name]
@@ -201,6 +180,21 @@ export class ShellEnv {
     return Promise.resolve(this.options.tmpDirs ?? ['/tmp']);
   }
 
+  async variables(names) {
+    const result = await this.sh(
+      'for n do printf "%s\\0" "$n"; printenv "$n" | tr -d "\\n"; printf "\\0"; done',
+      names
+    );
+    const fields = result.stdout.split('\0');
+    const vars = {};
+    for (let index = 0; index + 1 < fields.length; index += 2) {
+      if (fields[index + 1]) {
+        vars[fields[index]] = fields[index + 1];
+      }
+    }
+    return vars;
+  }
+
   async exists(target) {
     const result = await this.sh('[ -e "$1" ] || [ -L "$1" ]', [target]);
     return result.code === 0;
@@ -223,26 +217,7 @@ export class ShellEnv {
   }
 
   async stat(target) {
-    const result = await this.executor.run([
-      'stat',
-      '-c',
-      STAT_FORMAT,
-      '--',
-      target,
-    ]);
-    if (result.code !== 0) {
-      return null;
-    }
-    const parsed = parseStatLine(result.stdout.trim());
-    if (!parsed) {
-      return null;
-    }
-    return {
-      type: parsed.type,
-      size: parsed.size,
-      bytes: parsed.bytes,
-      mtimeMs: parsed.mtimeMs,
-    };
+    return (await this.statMany([target])).get(target) ?? null;
   }
 
   async readText(target, maxBytes = 1024 * 1024) {
@@ -317,7 +292,7 @@ export class ShellEnv {
       const result = await this.executor.run([
         'stat',
         '-c',
-        `${STAT_FORMAT}|%d:%i:%u`,
+        `${STAT_FORMAT}|%d:%i:%u:%g:%f`,
         '--',
         ...batch,
       ]);
@@ -330,6 +305,11 @@ export class ShellEnv {
             ...stat,
             inode: identity.slice(0, 2).join(':'),
             uid: identity.length > 2 ? Number(identity[2]) : undefined,
+            gid: identity.length > 3 ? Number(identity[3]) : undefined,
+            mode:
+              identity.length > 4
+                ? Number.parseInt(identity[4], 16)
+                : undefined,
           });
         }
       }
@@ -342,31 +322,24 @@ export class ShellEnv {
     const policy = this.scanPolicy;
     if (policy) {
       targets = targets.filter((target) => !policy.removalReason(target));
-      for (const target of targets) {
-        const expression = this.#boundaryPatterns().flatMap(
-          (pattern, index) => [
-            ...(index ? ['-o'] : []),
-            /[/\\]/.test(pattern) ? '-path' : '-name',
-            pattern.replaceAll('\\', '/'),
-          ]
-        );
-        const found = await this.executor.run([
-          'find',
-          target,
-          '-xdev',
-          '-mindepth',
-          '1',
-          '(',
-          ...expression,
-          ')',
-          '-print',
-          '-quit',
-        ]);
+      const expression = this.#boundaryPatterns().flatMap((pattern, index) => [
+        ...(index ? ['-o'] : []),
+        /[/\\]/.test(pattern) ? '-path' : '-name',
+        pattern.replaceAll('\\', '/'),
+      ]);
+      const script = `for t in "$@"; do
+        [ -e "$t" ] || [ -L "$t" ] || continue
+        find "$t" -xdev -mindepth 1 '(' ${expression.map(shellQuote).join(' ')} ')' -print -quit || printf '%s\\n' "$t"
+      done`;
+      for (const batch of batches(targets)) {
+        const found = await this.sh(script, batch);
         for (const line of found.stdout.split('\n').filter(Boolean)) {
           policy.excludedPaths.add(line);
         }
         if (found.code !== 0) {
-          policy.excludedPaths.add(target);
+          for (const target of batch) {
+            policy.excludedPaths.add(target);
+          }
         }
       }
       // du cannot prune same-device mounts or excluded children. Skip their
@@ -388,8 +361,21 @@ export class ShellEnv {
   }
 
   async realPath(target) {
-    const result = await this.executor.run(['readlink', '-f', '--', target]);
-    return result.code === 0 ? result.stdout.trim() || null : null;
+    return (await this.realPathMany([target])).get(target) ?? null;
+  }
+
+  async realPathMany(targets) {
+    const resolved = new Map();
+    for (const batch of batches(targets)) {
+      const result = await this.sh(
+        `for t do printf '${RECORD}%s\\n' "$t"; readlink -f -- "$t"; done`,
+        batch
+      );
+      for (const [target, body] of splitRecords(result.stdout)) {
+        resolved.set(target, body.trim() || null);
+      }
+    }
+    return resolved;
   }
 
   async isRoot() {
@@ -419,7 +405,47 @@ export class ShellEnv {
   }
 
   findFiles(roots, { names, maxDepth = 6, skipNames } = {}) {
-    return this.#find(roots, 'f', names, maxDepth, skipNames);
+    return this.#find(roots, 'f', names, maxDepth, [
+      ...new Set([
+        ...(skipNames ?? this.skipNames),
+        'node_modules',
+        'target',
+        'vendor',
+        '.venv',
+        'venv',
+      ]),
+    ]);
+  }
+
+  async projectActivity(project, globs) {
+    const prune = [
+      '-type',
+      'd',
+      '(',
+      ...findNameExpression([...new Set([...this.skipNames, ...globs])]),
+      ')',
+      '-prune',
+      '-o',
+    ];
+    const argv = [
+      'find',
+      project,
+      '-xdev',
+      ...this.#prunePaths(),
+      ...prune,
+      '-type',
+      'f',
+      '-exec',
+      'stat',
+      '-c',
+      '%Y',
+      '{}',
+      '+',
+    ];
+    const result = await this.sh(
+      `${argv.map(shellQuote).join(' ')} | awk 'BEGIN{m=0} {if ($1>m) m=$1} END{print m}'`
+    );
+    return Number(result.stdout.trim()) * 1000 || 0;
   }
 
   /**
@@ -544,6 +570,29 @@ export class ShellEnv {
       OPEN_PATHS_SCRIPT,
     ]);
     this.probeDiagnostic = null;
+    const probe = await this.#privilegedProbe(result);
+    result = probe.result;
+    let evidence = probe.evidence;
+    if (
+      evidence.unreadable.length &&
+      (this.probeExecutor || (await this.isRoot()))
+    ) {
+      evidence = await retryProcessOwners(
+        this.probeExecutor ?? this.executor,
+        evidence
+      );
+    }
+    this.unreadableProcesses = evidence.unreadable;
+    this.probeHint = inspectionHint(this.unreadableProcesses);
+    if (result.code !== 0) {
+      this.probeDiagnostic ??= result.stderr.trim();
+    }
+    return probe.coverageKnown ? evidence.paths : null;
+  }
+
+  async #privilegedProbe(result) {
+    const evidence = parseProcessProbe(result);
+    let coverageKnown = result.code === 0 || evidence.unreadable.length > 0;
     if (
       result.code !== 0 &&
       this.executor.parent &&
@@ -555,44 +604,33 @@ export class ShellEnv {
         this.id,
         diagnostic
       );
-      const inspector = containerExecutor(
-        this.executor.parent,
-        this.executor.containerId,
-        { user: '0', privileged: true }
+      const inspector = signalExecutor(
+        containerExecutor(
+          this.executor.parent.unscoped ?? this.executor.parent,
+          this.executor.containerId,
+          { user: '0', privileged: true }
+        ),
+        this.signal,
+        this.deadline
       );
-      result = await inspector.run(['sh', '-c', OPEN_PATHS_SCRIPT]);
-      if (result.code !== 0) {
-        this.probeDiagnostic = `${diagnostic}; privileged inspection failed: ${result.stderr.trim()}`;
-      } else {
+      const retried = await inspector.run(['sh', '-c', OPEN_PATHS_SCRIPT]);
+      const next = parseProcessProbe(retried);
+      for (const target of next.paths) {
+        evidence.paths.add(target);
+      }
+      if (retried.code === 0 || next.unreadable.length) {
+        coverageKnown = true;
+        evidence.unreadable = next.unreadable;
+        result = retried;
         this.probeExecutor = inspector;
       }
-    }
-    this.unreadableProcesses = result.stderr
-      .split('\n')
-      .filter((line) => line.startsWith('DSS_UNREADABLE|'))
-      .map((line) => {
-        const [, pid, uid, name, ...command] = line.split('|');
-        return {
-          pid: Number(pid),
-          uid: uid ? Number(uid) : null,
-          name,
-          command: command.join('|'),
-        };
-      });
-    if (result.code !== 0) {
-      this.probeDiagnostic ??= result.stderr.trim();
-      if (!this.unreadableProcesses.length) {
-        return null;
+      if (retried.code !== 0) {
+        this.probeDiagnostic = `${diagnostic}; privileged inspection failed: ${retried.stderr.trim()}`;
+      } else {
+        this.probeExecutor = inspector;
       }
-      this.probeHint =
-        'some processes could not be inspected; run as root for full verification';
     }
-    return new Set(
-      result.stdout
-        .split('\n')
-        .filter((line) => line.startsWith('/'))
-        .map((line) => line.replace(/ \(deleted\)$/, ''))
-    );
+    return { result, evidence, coverageKnown };
   }
 
   run(argv, options) {

package/src/exec.js

@@ -41,7 +41,15 @@ export function trace(...parts) {
  * @returns {Promise<{code: number, stdout: string, stderr: string, error?: Error}>}
  */
 export function runProcess(argv, options = {}) {
-  const { input, timeoutMs = DEFAULT_TIMEOUT_MS, cwd, env } = options;
+  const { input, timeoutMs = DEFAULT_TIMEOUT_MS, cwd, env, signal } = options;
+  if (signal?.aborted) {
+    return Promise.resolve({
+      code: 130,
+      stdout: '',
+      stderr: 'aborted',
+      aborted: true,
+    });
+  }
   trace('exec', JSON.stringify(argv));
   return new Promise((resolve) => {
     let child;
@@ -51,6 +59,7 @@ export function runProcess(argv, options = {}) {
         env: env ? { ...process.env, ...env } : process.env,
         stdio: ['pipe', 'pipe', 'pipe'],
         windowsHide: true,
+        detached: process.platform !== 'win32',
       });
     } catch (error) {
       resolve({ code: 127, stdout: '', stderr: String(error), error });
@@ -66,12 +75,33 @@ export function runProcess(argv, options = {}) {
       if (!settled) {
         settled = true;
         clearTimeout(timer);
+        signal?.removeEventListener('abort', abort);
         trace(`exit ${result.code} after ${Date.now() - startedAt}ms`);
         resolve(result);
       }
     };
+    const kill = () => {
+      try {
+        if (process.platform !== 'win32') {
+          process.kill(-child.pid, 'SIGKILL');
+        } else {
+          child.kill('SIGKILL');
+        }
+      } catch {
+        child.kill('SIGKILL');
+      }
+    };
+    const abort = () => {
+      kill();
+      finish({
+        code: 130,
+        stdout: Buffer.concat(stdout).toString('utf8'),
+        stderr: 'aborted',
+        aborted: true,
+      });
+    };
     const timer = setTimeout(() => {
-      child.kill('SIGKILL');
+      kill();
       finish({
         code: 124,
         stdout: Buffer.concat(stdout).toString('utf8'),
@@ -79,6 +109,10 @@ export function runProcess(argv, options = {}) {
         error: new Error(`timed out after ${timeoutMs}ms`),
       });
     }, timeoutMs);
+    signal?.addEventListener('abort', abort, { once: true });
+    if (signal?.aborted) {
+      abort();
+    }
     const collect = (target) => (chunk) => {
       captured += chunk.length;
       if (captured <= captureLimit) {
@@ -117,6 +151,11 @@ export function runProcess(argv, options = {}) {
  * @param {{timeoutMs?: number, cwd?: string}} [options]
  */
 export async function pipeInto(source, argv, options = {}) {
+  const abortSource = () => source.kill?.('SIGKILL');
+  options.signal?.addEventListener('abort', abortSource, { once: true });
+  if (options.signal?.aborted) {
+    abortSource();
+  }
   const sourceErr = [];
   source.stderr?.on('data', (chunk) => sourceErr.push(chunk));
   const sourceDone = new Promise((resolve) => {
@@ -124,7 +163,11 @@ export async function pipeInto(source, argv, options = {}) {
     source.on('close', (code) => resolve({ code: code ?? 1 }));
   });
   const result = await runProcess(argv, { ...options, stdin: source.stdout });
+  if (result.code !== 0) {
+    abortSource();
+  }
   const upstream = await sourceDone;
+  options.signal?.removeEventListener('abort', abortSource);
   if (upstream.code !== 0) {
     return {
       ...result,
@@ -143,11 +186,7 @@ export function hostExecutor() {
     label: 'host',
     depth: 0,
     run: (argv, options) => runProcess(argv, options),
-    spawn: (argv) =>
-      spawn(argv[0], argv.slice(1), {
-        stdio: ['ignore', 'pipe', 'pipe'],
-        windowsHide: true,
-      }),
+    spawn: spawnStream,
   };
 }
 
@@ -175,10 +214,42 @@ export function containerExecutor(parent, containerId, options = {}) {
     parent,
     run: (argv, runOptions = {}) =>
       parent.run(wrap(argv, runOptions.input !== undefined), runOptions),
-    spawn: (argv) => parent.spawn(wrap(argv, false)),
+    spawn: (argv, spawnOptions) =>
+      parent.spawn(wrap(argv, false), spawnOptions),
   };
 }
 
+/** Stream commands (Docker logs/cp) obey the same cancellation and deadline. */
+function spawnStream(argv, options = {}) {
+  const child = spawn(argv[0], argv.slice(1), {
+    stdio: ['ignore', 'pipe', 'pipe'],
+    windowsHide: true,
+    detached: process.platform !== 'win32',
+  });
+  const stop = () => {
+    try {
+      if (process.platform !== 'win32') {
+        process.kill(-child.pid, 'SIGKILL');
+      } else {
+        child.kill('SIGKILL');
+      }
+    } catch {
+      child.kill('SIGKILL');
+    }
+  };
+  const timer = setTimeout(stop, options.timeoutMs ?? 600000);
+  timer.unref?.();
+  options.signal?.addEventListener('abort', stop, { once: true });
+  if (options.signal?.aborted) {
+    stop();
+  }
+  child.once('close', () => {
+    clearTimeout(timer);
+    options.signal?.removeEventListener('abort', stop);
+  });
+  return child;
+}
+
 /**
  * Quotes a value for inclusion in a POSIX shell script.
  * @param {string} value