disk-space-saviour 0.0.0-stage → 0.12.0

package/src/env/resolve.js

@@ -0,0 +1,36 @@
+/**
+ * Rebuilds environment adapters from report descriptors, so a JSON report
+ * written by `dss scan --json` can be cleaned later: an empty chain is the
+ * host, every chain link is one `docker exec` hop.
+ */
+
+import { containerExecutor, hostExecutor } from '../exec.js';
+import { LocalEnv } from './local.js';
+import { ShellEnv } from './shell.js';
+
+/**
+ * @param {{id: string, label?: string, chain?: object[]}} descriptor
+ * @param {{env?: object}} [options] `env` overrides the host adapter
+ * @returns {{env: object, executor: object}}
+ */
+export function resolveEnvironment(descriptor, options = {}) {
+  const chain = descriptor.chain ?? [];
+  if (chain.length === 0) {
+    const env = options.env ?? new LocalEnv();
+    return { env, executor: env.executor ?? hostExecutor() };
+  }
+  const base = options.env?.executor ?? hostExecutor();
+  const executor = chain.reduce(
+    (parent, link) =>
+      containerExecutor(parent, link.containerId, {
+        label: `${parent.label}/${link.name}`,
+      }),
+    base
+  );
+  const env = new ShellEnv(executor, {
+    id: descriptor.id,
+    label: descriptor.label ?? executor.label,
+    kind: 'container',
+  });
+  return { env, executor };
+}

package/src/env/shell.js

@@ -0,0 +1,433 @@
+/**
+ * Environment adapter for a filesystem reached through an executor, usually
+ * a running container via `docker exec`.
+ *
+ * It needs only a POSIX shell with `find`, `stat -c`, `du`, `df`, `head`,
+ * `rm`, and `/proc` (GNU coreutils and busybox both qualify), so nothing is
+ * installed into the container being inspected.
+ */
+
+import path from 'node:path';
+
+import { shellQuote } from '../exec.js';
+import { DEFAULT_SKIP_NAMES, parseDf, processAliases } from './local.js';
+
+const STAT_FORMAT = '%F|%s|%b|%B|%Y|%n';
+const ARG_BATCH = 200;
+const RECORD = '\u0001';
+
+const LIST_SCRIPT = `for d in "$@"; do
+  printf '${RECORD}%s\\n' "$d"
+  cd -- "$d" 2>/dev/null && stat -c '${STAT_FORMAT}' -- * .[!.]* ..?* 2>/dev/null
+done
+true`;
+
+const HEADS_SCRIPT = `n="$1"; shift
+for f in "$@"; do
+  printf '${RECORD}%s\\n' "$f"
+  head -c "$n" -- "$f" 2>/dev/null
+done
+true`;
+
+const USAGE_MANY_SCRIPT = `for t in "$@"; do
+  [ -e "$t" ] || [ -L "$t" ] || continue
+  printf '${RECORD}%s\\n' "$t"
+  du -sk -- "$t" 2>/dev/null | cut -f1
+  find "$t" ! -type d -exec stat -c %Y {} + 2>/dev/null | awk 'BEGIN{m=0;n=0}{n++; if ($1>m) m=$1} END{print n, m}'
+  stat -c %Y -- "$t"
+done
+true`;
+
+const PROCESS_SCRIPT = `echo "$$"
+for p in /proc/[0-9]*; do
+  c=$(cat "$p/comm" 2>/dev/null) || continue
+  e=$(readlink "$p/exe" 2>/dev/null)
+  a=$(tr '\\000' '\\n' < "$p/cmdline" 2>/dev/null | head -n 1)
+  w=$(readlink "$p/cwd" 2>/dev/null)
+  t=$(sed 's/.*) //' "$p/stat" 2>/dev/null | cut -d' ' -f1,20)
+  printf '%s|%s|%s|%s|%s|%s\\n' "\${p#/proc/}" "$t" "$c" "$e" "$a" "$w"
+done`;
+
+const OPEN_PATHS_SCRIPT = `failed=0
+for p in /proc/[0-9]*; do
+  [ -d "$p" ] || continue
+  ls "$p/fd" >/dev/null 2>&1 || { [ -d "$p" ] && failed=1; continue; }
+  for l in "$p/cwd" "$p/exe" "$p"/fd/*; do
+    [ -L "$l" ] || continue
+    readlink "$l" 2>/dev/null || failed=1
+  done
+done
+exit "$failed"`;
+
+const REMOVE_SCRIPT = `rm -rf -- "$@" 2>/dev/null && exit 0
+chmod -R u+w -- "$@" 2>/dev/null
+rm -rf -- "$@"`;
+
+const EXISTS_SCRIPT = `for p in "$@"; do
+  if [ -e "$p" ] || [ -L "$p" ]; then printf '%s\\n' "$p"; fi
+done`;
+
+const HOMES_SCRIPT = `echo "$HOME"
+while IFS=: read -r name pw uid gid gecos home shell; do
+  case "$uid" in ''|*[!0-9]*) continue ;; esac
+  if [ "$uid" -eq 0 ] || { [ "$uid" -ge 1000 ] && [ "$uid" -lt 65534 ]; }; then
+    [ -d "$home" ] && [ "$home" != / ] && echo "$home"
+  fi
+done < /etc/passwd
+for h in /home/*; do [ -d "$h" ] && echo "$h"; done
+true`;
+
+function statType(label) {
+  if (label === 'directory') {
+    return 'dir';
+  }
+  if (label.startsWith('regular')) {
+    return 'file';
+  }
+  if (label === 'symbolic link') {
+    return 'symlink';
+  }
+  return 'other';
+}
+
+/**
+ * Parses one `stat -c '%F|%s|%b|%B|%Y|%n'` line.
+ * @param {string} line
+ */
+export function parseStatLine(line) {
+  const parts = line.split('|');
+  if (parts.length < 6) {
+    return null;
+  }
+  const [type, size, blocks, blockSize, mtime] = parts;
+  const name = parts.slice(5).join('|');
+  return {
+    name,
+    type: statType(type),
+    size: Number(size),
+    bytes: Number(blocks) * Number(blockSize),
+    mtimeMs: Number(mtime) * 1000,
+  };
+}
+
+function splitRecords(output) {
+  const records = new Map();
+  for (const chunk of output.split(RECORD).slice(1)) {
+    const newline = chunk.indexOf('\n');
+    const key = newline === -1 ? chunk : chunk.slice(0, newline);
+    records.set(key, newline === -1 ? '' : chunk.slice(newline + 1));
+  }
+  return records;
+}
+
+function parseUsage(body) {
+  const [kib, counts = '', rootMtime] = body.trim().split('\n');
+  const [files, newest] = counts.trim().split(/\s+/).map(Number);
+  const bytes = Number(kib) * 1024;
+  const newestSeconds = files > 0 ? newest : Number(rootMtime);
+  return {
+    bytes: Number.isFinite(bytes) ? bytes : 0,
+    apparentBytes: Number.isFinite(bytes) ? bytes : 0,
+    files: Number.isFinite(files) ? files : 0,
+    errors: Number.isFinite(bytes) ? 0 : 1,
+    newestMtimeMs: Number.isFinite(newestSeconds) ? newestSeconds * 1000 : 0,
+  };
+}
+
+function batches(items, size = ARG_BATCH) {
+  const result = [];
+  for (let index = 0; index < items.length; index += size) {
+    result.push(items.slice(index, index + size));
+  }
+  return result;
+}
+
+function findNameExpression(names) {
+  return names.flatMap((name, index) =>
+    index === 0 ? ['-name', name] : ['-o', '-name', name]
+  );
+}
+
+export class ShellEnv {
+  /**
+   * @param {object} executor executor reaching the target filesystem
+   * @param {{id?: string, label?: string, kind?: string, homes?: string[],
+   *   skipNames?: string[]}} [options]
+   */
+  constructor(executor, options = {}) {
+    this.executor = executor;
+    this.id = options.id ?? executor.label;
+    this.kind = options.kind ?? 'container';
+    this.label = options.label ?? executor.label;
+    this.path = path.posix;
+    this.platform = 'linux';
+    this.options = options;
+    this.vars = {};
+    this.currentHome = null;
+    this.skipNames = options.skipNames ?? DEFAULT_SKIP_NAMES;
+  }
+
+  sh(script, args = [], options = {}) {
+    return this.executor.run(['sh', '-c', script, 'sh', ...args], options);
+  }
+
+  async homeDirs() {
+    if (this.options.homes) {
+      return this.options.homes;
+    }
+    const result = await this.sh(HOMES_SCRIPT);
+    const lines = result.stdout.split('\n').filter(Boolean);
+    this.currentHome = lines[0] ?? null;
+    return [...new Set(lines.filter((line) => line.startsWith('/')))];
+  }
+
+  tmpDirs() {
+    return Promise.resolve(this.options.tmpDirs ?? ['/tmp']);
+  }
+
+  async exists(target) {
+    const result = await this.sh('[ -e "$1" ] || [ -L "$1" ]', [target]);
+    return result.code === 0;
+  }
+
+  /**
+   * Existing paths among `targets`, one round trip per batch.
+   * @returns {Promise<Set<string>>}
+   */
+  async existsMany(targets) {
+    const existing = new Set();
+    for (const batch of batches(targets)) {
+      const result = await this.sh(EXISTS_SCRIPT, batch);
+      result.stdout
+        .split('\n')
+        .filter(Boolean)
+        .forEach((line) => existing.add(line));
+    }
+    return existing;
+  }
+
+  async stat(target) {
+    const result = await this.executor.run([
+      'stat',
+      '-c',
+      STAT_FORMAT,
+      '--',
+      target,
+    ]);
+    if (result.code !== 0) {
+      return null;
+    }
+    const parsed = parseStatLine(result.stdout.trim());
+    if (!parsed) {
+      return null;
+    }
+    return {
+      type: parsed.type,
+      size: parsed.size,
+      bytes: parsed.bytes,
+      mtimeMs: parsed.mtimeMs,
+    };
+  }
+
+  async readText(target, maxBytes = 1024 * 1024) {
+    const result = await this.executor.run([
+      'head',
+      '-c',
+      String(maxBytes),
+      '--',
+      target,
+    ]);
+    return result.code === 0 ? result.stdout : null;
+  }
+
+  async readHeads(files, maxBytes = 4096) {
+    const heads = new Map();
+    for (const batch of batches(files)) {
+      const result = await this.sh(HEADS_SCRIPT, [String(maxBytes), ...batch]);
+      for (const [file, text] of splitRecords(result.stdout)) {
+        heads.set(file, text);
+      }
+    }
+    return heads;
+  }
+
+  /**
+   * Lists many directories with one round trip per batch.
+   * @param {string[]} dirs
+   * @returns {Promise<Map<string, object[]>>}
+   */
+  async listMany(dirs) {
+    const listings = new Map();
+    for (const batch of batches(dirs)) {
+      const result = await this.sh(LIST_SCRIPT, batch);
+      for (const [dir, body] of splitRecords(result.stdout)) {
+        const entries = body
+          .split('\n')
+          .map(parseStatLine)
+          .filter(Boolean)
+          .map((entry) => ({
+            ...entry,
+            path: path.posix.join(dir, entry.name),
+          }));
+        listings.set(dir, entries);
+      }
+    }
+    return listings;
+  }
+
+  async list(dir) {
+    return (await this.listMany([dir])).get(dir) ?? [];
+  }
+
+  async usage(target) {
+    return (await this.usageMany([target])).get(target) ?? null;
+  }
+
+  /**
+   * Measures many paths with one round trip per batch.
+   * @param {string[]} targets
+   * @returns {Promise<Map<string, object>>}
+   */
+  async usageMany(targets) {
+    const usages = new Map();
+    for (const batch of batches(targets)) {
+      const result = await this.sh(USAGE_MANY_SCRIPT, batch);
+      for (const [target, body] of splitRecords(result.stdout)) {
+        usages.set(target, parseUsage(body));
+      }
+    }
+    return usages;
+  }
+
+  async readLink(target) {
+    const result = await this.executor.run(['readlink', '--', target]);
+    return result.code === 0 ? result.stdout.trim() : null;
+  }
+
+  async realPath(target) {
+    const result = await this.executor.run(['readlink', '-f', '--', target]);
+    return result.code === 0 ? result.stdout.trim() || null : null;
+  }
+
+  async isRoot() {
+    if (this.root === undefined) {
+      const result = await this.executor.run(['id', '-u']);
+      this.root = result.stdout.trim() === '0';
+    }
+    return this.root;
+  }
+
+  async findDirs(roots, { globs, maxDepth = 6, skipNames } = {}) {
+    const matches = await this.#find(roots, 'd', globs, maxDepth, skipNames);
+    const parents = [...new Set(matches.map((m) => path.posix.dirname(m)))];
+    const listings = await this.listMany(parents);
+    return matches.map((match) => {
+      const parent = path.posix.dirname(match);
+      const siblings = (listings.get(parent) ?? []).map(
+        ({ name, type, mtimeMs }) => ({ name, type, mtimeMs })
+      );
+      return {
+        path: match,
+        name: path.posix.basename(match),
+        parent,
+        siblings,
+      };
+    });
+  }
+
+  findFiles(roots, { names, maxDepth = 6, skipNames } = {}) {
+    return this.#find(roots, 'f', names, maxDepth, skipNames);
+  }
+
+  /**
+   * Runs one `find` that prints entries of `type` matching `globs` (without
+   * descending into matched directories) and prunes `skipNames`.
+   */
+  async #find(roots, type, globs, maxDepth, skipNames) {
+    if (roots.length === 0 || globs.length === 0) {
+      return [];
+    }
+    const skip = skipNames ?? this.skipNames ?? [];
+    const pruneSkipped =
+      skip.length > 0
+        ? ['-type', 'd', '(', ...findNameExpression(skip), ')', '-prune', '-o']
+        : [];
+    const argv = [
+      'find',
+      ...roots,
+      '-mindepth',
+      '1',
+      '-maxdepth',
+      String(maxDepth),
+      ...(type === 'd'
+        ? ['-type', 'd', '(', ...findNameExpression(globs), ')', '-print']
+        : []),
+      ...(type === 'd' ? ['-prune', '-o'] : []),
+      ...pruneSkipped,
+      ...(type === 'f'
+        ? ['-type', 'f', '(', ...findNameExpression(globs), ')', '-print']
+        : ['-name', '']),
+    ];
+    const result = await this.executor.run(argv);
+    return result.stdout.split('\n').filter(Boolean);
+  }
+
+  async remove(targets) {
+    for (const batch of batches(targets)) {
+      const result = await this.sh(REMOVE_SCRIPT, batch);
+      if (result.code !== 0) {
+        throw new Error(
+          `rm failed in ${this.label}: ${result.stderr.trim() || result.code}`
+        );
+      }
+    }
+  }
+
+  async processes() {
+    const result = await this.sh(PROCESS_SCRIPT);
+    if (result.code !== 0) {
+      return null;
+    }
+    const [selfPid, ...lines] = result.stdout.split('\n');
+    return lines
+      .map((line) =>
+        /^(\d+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)\|(.*)$/.exec(line)
+      )
+      .filter((m) => m && !m[2].startsWith('Z '))
+      .map((m) => ({
+        pid: Number(m[1]),
+        startTime: m[2].split(' ')[1] || null,
+        name: m[3],
+        aliases: processAliases(m[4], m[5]),
+        cwd: m[6] || null,
+      }))
+      .filter((proc) => String(proc.pid) !== selfPid.trim());
+  }
+
+  async openPaths() {
+    const result = await this.sh(OPEN_PATHS_SCRIPT);
+    if (result.code !== 0) {
+      return null;
+    }
+    return new Set(
+      result.stdout
+        .split('\n')
+        .filter((line) => line.startsWith('/'))
+        .map((line) => line.replace(/ \(deleted\)$/, ''))
+    );
+  }
+
+  run(argv, options) {
+    return this.executor.run(argv, options);
+  }
+
+  async which(command) {
+    const result = await this.sh(`command -v ${shellQuote(command)}`);
+    return result.code === 0 && result.stdout.trim() !== '';
+  }
+
+  async diskUsage(target) {
+    const result = await this.executor.run(['df', '-Pk', target]);
+    return result.code === 0 ? parseDf(result.stdout) : null;
+  }
+}

package/src/exec.js

@@ -0,0 +1,185 @@
+/**
+ * Command execution primitives.
+ *
+ * An executor runs argv arrays somewhere: on the host, or inside a container
+ * through `docker exec`. Executors chain, so a container inside a
+ * Docker-in-Docker daemon is reached by wrapping the outer container's
+ * executor. Every higher layer (filesystem adapters, Docker client, git
+ * checks) talks to an executor and never cares how deep it is.
+ */
+
+import { spawn } from 'node:child_process';
+
+const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
+const MAX_CAPTURE_BYTES = 256 * 1024 * 1024;
+
+/**
+ * Debug tracing, off by default. Enable with DSS_DEBUG=1 or `--verbose`.
+ */
+let traceEnabled = false;
+try {
+  traceEnabled = Boolean(globalThis.process?.env?.DSS_DEBUG);
+} catch {
+  traceEnabled = false;
+}
+
+export function setTrace(enabled) {
+  traceEnabled = Boolean(enabled);
+}
+
+export function trace(...parts) {
+  if (traceEnabled) {
+    console.error('[dss]', ...parts);
+  }
+}
+
+/**
+ * Spawns argv and captures its output.
+ * @param {string[]} argv
+ * @param {{input?: string, stdin?: import('node:stream').Readable,
+ *   timeoutMs?: number, cwd?: string, env?: object}} [options]
+ * @returns {Promise<{code: number, stdout: string, stderr: string, error?: Error}>}
+ */
+export function runProcess(argv, options = {}) {
+  const { input, timeoutMs = DEFAULT_TIMEOUT_MS, cwd, env } = options;
+  trace('exec', JSON.stringify(argv));
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(argv[0], argv.slice(1), {
+        cwd,
+        env: env ? { ...process.env, ...env } : process.env,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        windowsHide: true,
+      });
+    } catch (error) {
+      resolve({ code: 127, stdout: '', stderr: String(error), error });
+      return;
+    }
+    const stdout = [];
+    const stderr = [];
+    let captured = 0;
+    let settled = false;
+    const startedAt = Date.now();
+    const finish = (result) => {
+      if (!settled) {
+        settled = true;
+        clearTimeout(timer);
+        trace(`exit ${result.code} after ${Date.now() - startedAt}ms`);
+        resolve(result);
+      }
+    };
+    const timer = setTimeout(() => {
+      child.kill('SIGKILL');
+      finish({
+        code: 124,
+        stdout: Buffer.concat(stdout).toString('utf8'),
+        stderr: `timed out after ${timeoutMs}ms`,
+        error: new Error(`timed out after ${timeoutMs}ms`),
+      });
+    }, timeoutMs);
+    const collect = (target) => (chunk) => {
+      captured += chunk.length;
+      if (captured <= MAX_CAPTURE_BYTES) {
+        target.push(chunk);
+      }
+    };
+    child.stdout.on('data', collect(stdout));
+    child.stderr.on('data', collect(stderr));
+    child.on('error', (error) => {
+      finish({ code: 127, stdout: '', stderr: String(error), error });
+    });
+    child.on('close', (code) => {
+      finish({
+        code: code ?? 1,
+        stdout: Buffer.concat(stdout).toString('utf8'),
+        stderr: Buffer.concat(stderr).toString('utf8'),
+      });
+    });
+    child.stdin.on('error', () => {});
+    if (options.stdin) {
+      options.stdin.pipe(child.stdin);
+    } else if (input !== undefined) {
+      child.stdin.end(input);
+    } else {
+      child.stdin.end();
+    }
+  });
+}
+
+/**
+ * Pipes the stdout of an already spawned `source` process into `argv`.
+ * The result fails when either side fails.
+ * @param {import('node:child_process').ChildProcess} source
+ * @param {string[]} argv
+ * @param {{timeoutMs?: number, cwd?: string}} [options]
+ */
+export async function pipeInto(source, argv, options = {}) {
+  const sourceErr = [];
+  source.stderr?.on('data', (chunk) => sourceErr.push(chunk));
+  const sourceDone = new Promise((resolve) => {
+    source.on('error', (error) => resolve({ code: 127, error }));
+    source.on('close', (code) => resolve({ code: code ?? 1 }));
+  });
+  const result = await runProcess(argv, { ...options, stdin: source.stdout });
+  const upstream = await sourceDone;
+  if (upstream.code !== 0) {
+    return {
+      ...result,
+      code: upstream.code,
+      stderr: `${Buffer.concat(sourceErr).toString('utf8')}${upstream.error ?? ''}${result.stderr}`,
+    };
+  }
+  return result;
+}
+
+/**
+ * Executor that runs commands on the machine running dss.
+ */
+export function hostExecutor() {
+  return {
+    label: 'host',
+    depth: 0,
+    run: (argv, options) => runProcess(argv, options),
+    spawn: (argv) =>
+      spawn(argv[0], argv.slice(1), {
+        stdio: ['ignore', 'pipe', 'pipe'],
+        windowsHide: true,
+      }),
+  };
+}
+
+/**
+ * Executor that runs commands inside a container via `docker exec`.
+ * @param {object} parent executor that can reach the container's daemon
+ * @param {string} containerId
+ * @param {{docker?: string, label?: string, user?: string}} [options]
+ */
+export function containerExecutor(parent, containerId, options = {}) {
+  const docker = options.docker ?? 'docker';
+  const wrap = (argv, interactive) => [
+    docker,
+    'exec',
+    ...(interactive ? ['-i'] : []),
+    ...(options.user ? ['--user', options.user] : []),
+    containerId,
+    ...argv,
+  ];
+  return {
+    label: options.label ?? `${parent.label}/${containerId.slice(0, 12)}`,
+    depth: (parent.depth ?? 0) + 1,
+    containerId,
+    parent,
+    run: (argv, runOptions = {}) =>
+      parent.run(wrap(argv, runOptions.input !== undefined), runOptions),
+    spawn: (argv) => parent.spawn(wrap(argv, false)),
+  };
+}
+
+/**
+ * Quotes a value for inclusion in a POSIX shell script.
+ * @param {string} value
+ */
+export function shellQuote(value) {
+  return `'${String(value).replaceAll("'", "'\\''")}'`;
+}