disk-space-saviour 0.13.1 → 0.15.1

package/src/env/shell.js

@@ -8,8 +8,9 @@
  */
 
 import path from 'node:path';
+import { sharedWritableLayer } from '../docker/writable.js';
 
-import { shellQuote } from '../exec.js';
+import { containerExecutor, shellQuote, trace } from '../exec.js';
 import { DEFAULT_SKIP_NAMES, parseDf, processAliases } from './local.js';
 
 const STAT_FORMAT = '%F|%s|%b|%B|%Y|%n';
@@ -32,8 +33,8 @@ true`;
 const USAGE_MANY_SCRIPT = `for t in "$@"; do
   [ -e "$t" ] || [ -L "$t" ] || continue
   printf '${RECORD}%s\\n' "$t"
-  du -sk -- "$t" 2>/dev/null | cut -f1
-  find "$t" ! -type d -exec stat -c %Y {} + 2>/dev/null | awk 'BEGIN{m=0;n=0}{n++; if ($1>m) m=$1} END{print n, m}'
+  du -skx -- "$t" 2>/dev/null | cut -f1
+  find "$t" -xdev ! -type d -exec stat -c %Y {} + 2>/dev/null | awk 'BEGIN{m=0;n=0}{n++; if ($1>m) m=$1} END{print n, m}'
   stat -c %Y -- "$t"
 done
 true`;
@@ -44,18 +45,29 @@ for p in /proc/[0-9]*; do
   e=$(readlink "$p/exe" 2>/dev/null)
   a=$(tr '\\000' '\\n' < "$p/cmdline" 2>/dev/null | head -n 1)
   w=$(readlink "$p/cwd" 2>/dev/null)
+  q=$(tr '\\000' ' ' < "$p/cmdline" 2>/dev/null)
   t=$(sed 's/.*) //' "$p/stat" 2>/dev/null | cut -d' ' -f1,20)
-  printf '%s|%s|%s|%s|%s|%s\\n' "\${p#/proc/}" "$t" "$c" "$e" "$a" "$w"
+  printf '%s|%s|%s|%s|%s|%s|%s\\n' "\${p#/proc/}" "$t" "$c" "$e" "$a" "$w" "$q"
 done`;
 
-const OPEN_PATHS_SCRIPT = `failed=0
+export const OPEN_PATHS_SCRIPT = `failed=0
 for p in /proc/[0-9]*; do
   [ -d "$p" ] || continue
-  ls "$p/fd" >/dev/null 2>&1 || { [ -d "$p" ] && failed=1; continue; }
+  state=$(sed 's/.*) //' "$p/stat" 2>/dev/null | cut -d' ' -f1)
+  [ "$state" = Z ] && continue
+  unreadable=0
+  ls "$p/fd" >/dev/null 2>&1 || unreadable=1
   for l in "$p/cwd" "$p/exe" "$p"/fd/*; do
     [ -L "$l" ] || continue
-    readlink "$l" 2>/dev/null || failed=1
+    readlink "$l" 2>/dev/null || { [ -L "$l" ] && unreadable=1; }
   done
+  if [ "$unreadable" = 1 ] && [ -d "$p" ]; then
+    printf 'unreadable pid %s (%s)\\n' "\${p#/proc/}" "$(cat "$p/comm" 2>/dev/null)" >&2
+    uid=$(awk '/^Uid:/{print $2}' "$p/status" 2>/dev/null)
+    command=$(tr '\\000' ' ' < "$p/cmdline" 2>/dev/null)
+    printf 'DSS_UNREADABLE|%s|%s|%s|%s\\n' "\${p#/proc/}" "$uid" "$(cat "$p/comm" 2>/dev/null)" "$command" >&2
+    failed=1
+  fi
 done
 exit "$failed"`;
 
@@ -156,6 +168,10 @@ export class ShellEnv {
    */
   constructor(executor, options = {}) {
     this.executor = executor;
+    this.writableLayer =
+      executor.parent && executor.containerId
+        ? sharedWritableLayer(executor.parent, executor.containerId)
+        : null;
     this.id = options.id ?? executor.label;
     this.kind = options.kind ?? 'container';
     this.label = options.label ?? executor.label;
@@ -289,7 +305,74 @@ export class ShellEnv {
    * @returns {Promise<Map<string, object>>}
    */
   async usageMany(targets) {
+    const usages = await this.rawUsageMany(targets);
+    return this.writableLayer
+      ? this.writableLayer.measure(this, usages)
+      : usages;
+  }
+
+  async statMany(targets) {
+    const stats = new Map();
+    for (const batch of batches(targets)) {
+      const result = await this.executor.run([
+        'stat',
+        '-c',
+        `${STAT_FORMAT}|%d:%i:%u`,
+        '--',
+        ...batch,
+      ]);
+      for (const line of result.stdout.split('\n').filter(Boolean)) {
+        const end = line.lastIndexOf('|');
+        const stat = parseStatLine(line.slice(0, end));
+        if (stat) {
+          const identity = line.slice(end + 1).split(':');
+          stats.set(stat.name, {
+            ...stat,
+            inode: identity.slice(0, 2).join(':'),
+            uid: identity.length > 2 ? Number(identity[2]) : undefined,
+          });
+        }
+      }
+    }
+    return stats;
+  }
+
+  async rawUsageMany(targets) {
     const usages = new Map();
+    const policy = this.scanPolicy;
+    if (policy) {
+      targets = targets.filter((target) => !policy.removalReason(target));
+      for (const target of targets) {
+        const expression = this.#boundaryPatterns().flatMap(
+          (pattern, index) => [
+            ...(index ? ['-o'] : []),
+            /[/\\]/.test(pattern) ? '-path' : '-name',
+            pattern.replaceAll('\\', '/'),
+          ]
+        );
+        const found = await this.executor.run([
+          'find',
+          target,
+          '-xdev',
+          '-mindepth',
+          '1',
+          '(',
+          ...expression,
+          ')',
+          '-print',
+          '-quit',
+        ]);
+        for (const line of found.stdout.split('\n').filter(Boolean)) {
+          policy.excludedPaths.add(line);
+        }
+        if (found.code !== 0) {
+          policy.excludedPaths.add(target);
+        }
+      }
+      // du cannot prune same-device mounts or excluded children. Skip their
+      // ancestors before either recursive measurement command can enter them.
+      targets = targets.filter((target) => !policy.removalReason(target));
+    }
     for (const batch of batches(targets)) {
       const result = await this.sh(USAGE_MANY_SCRIPT, batch);
       for (const [target, body] of splitRecords(result.stdout)) {
@@ -344,6 +427,9 @@ export class ShellEnv {
    * descending into matched directories) and prunes `skipNames`.
    */
   async #find(roots, type, globs, maxDepth, skipNames) {
+    roots = roots.filter(
+      (root) => !this.scanPolicy || this.scanPolicy.allows(root)
+    );
     if (roots.length === 0 || globs.length === 0) {
       return [];
     }
@@ -355,10 +441,12 @@ export class ShellEnv {
     const argv = [
       'find',
       ...roots,
+      '-xdev',
       '-mindepth',
       '1',
       '-maxdepth',
       String(maxDepth),
+      ...this.#prunePaths(),
       ...(type === 'd'
         ? ['-type', 'd', '(', ...findNameExpression(globs), ')', '-print']
         : []),
@@ -369,7 +457,47 @@ export class ShellEnv {
         : ['-name', '']),
     ];
     const result = await this.executor.run(argv);
-    return result.stdout.split('\n').filter(Boolean);
+    return result.stdout
+      .split('\n')
+      .filter(
+        (target) =>
+          target && (!this.scanPolicy || this.scanPolicy.allows(target))
+      );
+  }
+
+  #boundaryPatterns() {
+    const policy = this.scanPolicy;
+    if (!policy) {
+      return [];
+    }
+    return [
+      ...policy.exclude,
+      ...policy.pruned,
+      '/var/lib/docker',
+      '/var/lib/containerd',
+      '/run/containerd',
+      '/var/lib/containers',
+      '/tmp/containerd-mount*',
+      '/private/tmp/containerd-mount*',
+    ].map((pattern) =>
+      // find's * also matches slashes. Match optional globstar segments
+      // conservatively so its traversal cannot miss a zero-depth exclusion.
+      pattern
+        .replaceAll('\\', '/')
+        .replace(/\/\*\*$/, '')
+        .replaceAll('**/', '*')
+    );
+  }
+
+  #prunePaths() {
+    return this.#boundaryPatterns().flatMap((pattern) => [
+      '(',
+      /[/\\]/.test(pattern) ? '-path' : '-name',
+      pattern,
+      ')',
+      '-prune',
+      '-o',
+    ]);
   }
 
   async remove(targets) {
@@ -384,14 +512,18 @@ export class ShellEnv {
   }
 
   async processes() {
-    const result = await this.sh(PROCESS_SCRIPT);
+    const result = await (this.probeExecutor ?? this.executor).run([
+      'sh',
+      '-c',
+      PROCESS_SCRIPT,
+    ]);
     if (result.code !== 0) {
       return null;
     }
     const [selfPid, ...lines] = result.stdout.split('\n');
     return lines
       .map((line) =>
-        /^(\d+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)\|(.*)$/.exec(line)
+        /^(\d+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)\|(.*)$/.exec(line)
       )
       .filter((m) => m && !m[2].startsWith('Z '))
       .map((m) => ({
@@ -400,14 +532,60 @@ export class ShellEnv {
         name: m[3],
         aliases: processAliases(m[4], m[5]),
         cwd: m[6] || null,
+        command: m[7],
       }))
       .filter((proc) => String(proc.pid) !== selfPid.trim());
   }
 
   async openPaths() {
-    const result = await this.sh(OPEN_PATHS_SCRIPT);
+    let result = await (this.probeExecutor ?? this.executor).run([
+      'sh',
+      '-c',
+      OPEN_PATHS_SCRIPT,
+    ]);
+    this.probeDiagnostic = null;
+    if (
+      result.code !== 0 &&
+      this.executor.parent &&
+      this.executor.containerId
+    ) {
+      const diagnostic = result.stderr.trim();
+      trace(
+        'retrying process inspection as privileged root',
+        this.id,
+        diagnostic
+      );
+      const inspector = containerExecutor(
+        this.executor.parent,
+        this.executor.containerId,
+        { user: '0', privileged: true }
+      );
+      result = await inspector.run(['sh', '-c', OPEN_PATHS_SCRIPT]);
+      if (result.code !== 0) {
+        this.probeDiagnostic = `${diagnostic}; privileged inspection failed: ${result.stderr.trim()}`;
+      } else {
+        this.probeExecutor = inspector;
+      }
+    }
+    this.unreadableProcesses = result.stderr
+      .split('\n')
+      .filter((line) => line.startsWith('DSS_UNREADABLE|'))
+      .map((line) => {
+        const [, pid, uid, name, ...command] = line.split('|');
+        return {
+          pid: Number(pid),
+          uid: uid ? Number(uid) : null,
+          name,
+          command: command.join('|'),
+        };
+      });
     if (result.code !== 0) {
-      return null;
+      this.probeDiagnostic ??= result.stderr.trim();
+      if (!this.unreadableProcesses.length) {
+        return null;
+      }
+      this.probeHint =
+        'some processes could not be inspected; run as root for full verification';
     }
     return new Set(
       result.stdout

package/src/exec.js

@@ -59,6 +59,7 @@ export function runProcess(argv, options = {}) {
     const stdout = [];
     const stderr = [];
     let captured = 0;
+    const captureLimit = options.maxCaptureBytes ?? MAX_CAPTURE_BYTES;
     let settled = false;
     const startedAt = Date.now();
     const finish = (result) => {
@@ -80,7 +81,7 @@ export function runProcess(argv, options = {}) {
     }, timeoutMs);
     const collect = (target) => (chunk) => {
       captured += chunk.length;
-      if (captured <= MAX_CAPTURE_BYTES) {
+      if (captured <= captureLimit) {
         target.push(chunk);
       }
     };
@@ -94,6 +95,7 @@ export function runProcess(argv, options = {}) {
         code: code ?? 1,
         stdout: Buffer.concat(stdout).toString('utf8'),
         stderr: Buffer.concat(stderr).toString('utf8'),
+        truncated: captured > captureLimit,
       });
     });
     child.stdin.on('error', () => {});
@@ -161,6 +163,7 @@ export function containerExecutor(parent, containerId, options = {}) {
     docker,
     'exec',
     ...(interactive ? ['-i'] : []),
+    ...(options.privileged ? ['--privileged'] : []),
     ...(options.user ? ['--user', options.user] : []),
     containerId,
     ...argv,

package/src/git-preserved.js

@@ -0,0 +1,120 @@
+/** Verify unstaged work against cached remote refs without running filters. */
+import { createHash } from 'node:crypto';
+import { createReadStream, promises as fsp } from 'node:fs';
+import path from 'node:path';
+
+async function blob(root, file, algorithm) {
+  const target = path.join(root, file);
+  const stat = await fsp.lstat(target).catch((error) => {
+    if (['ENOENT', 'ENOTDIR'].includes(error.code)) {
+      return null;
+    }
+    throw error;
+  });
+  if (!stat) {
+    return null;
+  }
+  if (!stat.isFile() && !stat.isSymbolicLink()) {
+    throw new Error('non-file change');
+  }
+  const link = stat.isSymbolicLink()
+    ? Buffer.from(await fsp.readlink(target))
+    : null;
+  const hash = createHash(algorithm);
+  hash.update(`blob ${link ? link.length : stat.size}\0`);
+  if (link) {
+    hash.update(link);
+  } else {
+    for await (const chunk of createReadStream(target)) {
+      hash.update(chunk);
+    }
+  }
+  return {
+    oid: hash.digest('hex'),
+    mode: link ? '120000' : stat.mode & 0o111 ? '100755' : '100644',
+  };
+}
+
+/** All changes must be preserved by one ref; staged changes remain protected. */
+export async function preservedWork(root, git) {
+  const status = await git([
+    'status',
+    '--porcelain=v1',
+    '-z',
+    '--untracked-files=all',
+  ]);
+  if (status.code !== 0) {
+    return {};
+  }
+  const records = status.stdout.split('\0').filter(Boolean);
+  const untracked = records.filter((line) => line.startsWith('?? ')).length;
+  const result = { untracked, dirty: records.length };
+  if (
+    !records.length ||
+    records.length > 2000 ||
+    records.some((line) => ![' ', '?'].includes(line[0]))
+  ) {
+    return result;
+  }
+  const refs = await git([
+    'for-each-ref',
+    '--format=%(refname)',
+    'refs/remotes',
+  ]);
+  const format = await git(['rev-parse', '--show-object-format']);
+  if (refs.code !== 0 || format.code !== 0) {
+    return result;
+  }
+  const algorithm = format.stdout.trim();
+  if (!['sha1', 'sha256'].includes(algorithm)) {
+    return result;
+  }
+  try {
+    const files = new Map();
+    for (const record of records) {
+      const file = record.slice(3);
+      files.set(file, await blob(root, file, algorithm));
+    }
+    for (const ref of refs.stdout
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .slice(0, 100)) {
+      const tree = await git([
+        'ls-tree',
+        '-r',
+        '-z',
+        ref,
+        '--',
+        ...[...files.keys()].map((file) => `:(literal)${file}`),
+      ]);
+      if (tree.code !== 0) {
+        continue;
+      }
+      const saved = new Map(
+        tree.stdout
+          .split('\0')
+          .filter(Boolean)
+          .map((line) => {
+            const tab = line.indexOf('\t');
+            const [mode, , oid] = line.slice(0, tab).split(' ');
+            return [line.slice(tab + 1), { mode, oid }];
+          })
+      );
+      if (
+        [...files].every(([file, current]) =>
+          current
+            ? saved.get(file)?.oid === current.oid &&
+              saved.get(file)?.mode === current.mode
+            : !saved.has(file)
+        )
+      ) {
+        result.preservedOn = ref.replace(/^refs\/remotes\//, '');
+        break;
+      }
+    }
+  } catch {
+    // Unreadable files and unsupported changes keep the original blocker.
+  }
+  return result;
+}

package/src/git.js

@@ -15,6 +15,7 @@ import os from 'node:os';
 import path from 'node:path';
 
 import { trace } from './exec.js';
+import { preservedWork } from './git-preserved.js';
 
 const GIT_FLAGS = [
   '--no-optional-locks',
@@ -93,7 +94,7 @@ export function stateBlockers(label, state) {
     return [`${label}: ${state.error}`];
   }
   const problems = [];
-  if (state.dirty > 0) {
+  if (state.dirty > 0 && !state.preservedOn) {
     problems.push(plural(state.dirty, 'uncommitted change'));
   }
   if (state.unpushed > 0) {
@@ -111,8 +112,9 @@ export class GitInspector {
   /**
    * @param {object} env environment adapter
    */
-  constructor(env) {
+  constructor(env, options = {}) {
     this.env = env;
+    this.options = options;
     this.states = new Map();
     this.roots = new Map();
     this.available = null;
@@ -224,6 +226,9 @@ export class GitInspector {
       return state;
     }
     state.dirty = countLines(status.stdout);
+    if (this.options.recognizePreserved && state.dirty > 0) {
+      Object.assign(state, await preservedWork(root, git));
+    }
     const unpushed = await git([
       'log',
       'HEAD',
@@ -259,7 +264,7 @@ export class GitInspector {
   }
 
   /** Protect an entire temporary work directory, including nested clones. */
-  async treeBlockers(target) {
+  async treeBlockers(target, { allowNoRepo = false } = {}) {
     const dirs = await this.env.findDirs([target], {
       globs: ['.git'],
       maxDepth: 32,
@@ -278,6 +283,9 @@ export class GitInspector {
       ...files.map((file) => this.env.path.dirname(file)),
     ]);
     if (roots.size === 0) {
+      if (allowNoRepo) {
+        return [];
+      }
       return [`${target}: no Git repository found; cannot verify work`];
     }
     if (roots.size > 50) {
@@ -303,9 +311,27 @@ export class GitInspector {
       const result =
         this.env.kind === 'host'
           ? await safeGitDirectory(root, (safeDir) =>
-              this.#git(root, ['ls-files', '--', relative], safeDir)
+              this.#git(
+                root,
+                [
+                  'ls-files',
+                  '--cached',
+                  '--others',
+                  '--exclude-standard',
+                  '--',
+                  relative,
+                ],
+                safeDir
+              )
             )
-          : await this.#git(root, ['ls-files', '--', relative]);
+          : await this.#git(root, [
+              'ls-files',
+              '--cached',
+              '--others',
+              '--exclude-standard',
+              '--',
+              relative,
+            ]);
       return result.code === 0 ? countLines(result.stdout) : -1;
     } catch {
       return -1;
@@ -331,7 +357,9 @@ export class GitInspector {
       return [`cannot verify Git tracking of ${target}`];
     }
     if (tracked > 0) {
-      return [`${plural(tracked, 'file')} inside ${target} tracked by Git`];
+      return [
+        `${plural(tracked, 'file')} inside ${target} tracked by Git or not ignored`,
+      ];
     }
     return requireClean ? this.repoBlockers(root) : [];
   }

package/src/index.d.ts

@@ -15,7 +15,7 @@ export interface ScanOptions {
   staleAge?: Duration;
   /** Alias of `staleAge`. */
   olderThan?: Duration;
-  /** Inactivity before whole dependency dirs become `moderate` (`30d`). */
+  /** Inactivity before lockfile-backed node_modules become `safe`, other dependency dirs `moderate` (`30d`). */
   inactive?: Duration;
   /** Items smaller than this are dropped (`1M`). */
   minSize?: Size;
@@ -25,7 +25,7 @@ export interface ScanOptions {
   docker?: boolean | null;
   /** Docker nesting depth (1 = containers of the host daemon). */
   dockerDepth?: number;
-  /** Container ids or names to limit Docker work to. */
+  /** Container ids, names or unique prefixes to limit Docker work to. */
   containers?: string[] | string;
   scanners?: Array<'projects' | 'global' | 'versions' | 'agents' | 'system'>;
   /** Only these ecosystems, rules or kinds. */
@@ -34,6 +34,8 @@ export interface ScanOptions {
   exclude?: string[] | string;
   includeVolumes?: boolean;
   removeStoppedContainers?: boolean;
+  /** Opt-in age after which failed containers may pass the investigation hold. */
+  investigationMaxAge?: Duration | null;
   /**
    * Stopped containers (name, id or id prefix of 12+ characters) the
    * operator approves one by one, including ones kept for investigation.
@@ -63,6 +65,7 @@ export interface CleanOptions extends ScanOptions {
   confirm?: (item: Item) => boolean | Promise<boolean>;
   /** `false` skips writing the audit log file. */
   audit?: boolean;
+  signal?: AbortSignal;
   onEntry?: (entry: AuditEntry, item: Item) => void;
 }
 
@@ -97,6 +100,12 @@ export interface Item {
   path: string | null;
   paths: string[];
   bytes: number;
+  /** Merged filesystem bytes, including image data, when measured in Docker. */
+  totalBytes?: number;
+  /** Immutable image bytes excluded from reclaimable totals. */
+  imageBytes?: number;
+  /** The object was discovered but its reclaimable size could not be measured. */
+  sizeUnknown?: boolean;
   newestMtimeMs: number;
   tier: Tier;
   reason: string;
@@ -134,6 +143,7 @@ export interface EnvironmentDescriptor {
   kind: 'host' | 'container';
   depth: number;
   chain: Array<{ containerId: string; name: string }>;
+  hint?: string | null;
   disk?: Disk | null;
 }
 
@@ -200,6 +210,8 @@ export interface AuditLog {
   dryRun: boolean;
   startedAt: string;
   finishedAt: string | null;
+  aborted?: boolean;
+  error?: string;
   entries: AuditEntry[];
   freedBytes: number;
   plannedBytes: number;