disk-space-saviour 0.0.0-stage → 0.11.30

package/src/docker/cli.js

@@ -0,0 +1,289 @@
+/**
+ * Docker CLI access through an executor (the host, or a container running
+ * its own daemon for Docker-in-Docker).
+ *
+ * Every invocation passes `assertAllowed`: dss never stops, kills,
+ * restarts, pauses or force-removes anything, and `docker rm` is only used
+ * for containers that are verified stopped right before removal.
+ */
+
+import { pipeInto } from '../exec.js';
+
+const DECIMAL_UNITS = {
+  b: 1,
+  kb: 1e3,
+  mb: 1e6,
+  gb: 1e9,
+  tb: 1e12,
+  pb: 1e15,
+  kib: 1024,
+  mib: 1024 ** 2,
+  gib: 1024 ** 3,
+  tib: 1024 ** 4,
+};
+
+const ALLOWED = new Map([
+  ['version', null],
+  ['info', null],
+  ['ps', null],
+  ['inspect', null],
+  ['diff', null],
+  ['logs', null],
+  ['cp', null],
+  ['history', null],
+  ['exec', null],
+  ['rm', 'rm'],
+  ['system', new Set(['df'])],
+  ['image', new Set(['prune', 'rm', 'ls', 'inspect', 'history'])],
+  ['builder', new Set(['prune', 'du'])],
+  ['volume', new Set(['ls', 'rm', 'inspect'])],
+  ['container', new Set(['inspect', 'ls', 'diff', 'logs'])],
+]);
+
+const FORCE_FLAGS = new Set(['-f', '--force', '-v', '--volumes', '-l']);
+
+function assertContainerRemoval(args) {
+  if (args.slice(1).some((arg) => FORCE_FLAGS.has(arg))) {
+    throw new Error('dss never force-removes containers or their volumes');
+  }
+  if (args.length !== 2 || args[1].startsWith('-')) {
+    throw new Error('dss only removes one container by id');
+  }
+}
+
+function assertImageMutation(sub, args) {
+  if (sub === 'rm' && (args.length !== 3 || args[2].startsWith('-'))) {
+    throw new Error('dss only removes one image by id');
+  }
+  if (sub === 'prune' && (args.length !== 3 || args[2] !== '--force')) {
+    throw new Error('dss only prunes dangling images');
+  }
+}
+
+/**
+ * Throws unless `args` (the arguments after `docker`) is a command dss may
+ * run. `docker exec` payloads are not inspected: they run inside a
+ * container, never against its lifecycle.
+ * @param {string[]} args
+ */
+export function assertAllowed(args) {
+  const [command, sub] = args;
+  if (!ALLOWED.has(command)) {
+    throw new Error(`dss refuses to run \`docker ${command}\``);
+  }
+  const rule = ALLOWED.get(command);
+  if (rule instanceof Set && !rule.has(sub)) {
+    throw new Error(`dss refuses to run \`docker ${command} ${sub}\``);
+  }
+  if (rule === 'rm') {
+    assertContainerRemoval(args);
+  }
+  if (command === 'volume' && sub === 'rm') {
+    throw new Error('dss never removes Docker volumes automatically');
+  }
+  if (command === 'image') {
+    assertImageMutation(sub, args);
+  }
+}
+
+/**
+ * Parses sizes printed by Docker (`19.1MB`, `0B`, `1.2GB (virtual 3GB)`),
+ * which use decimal multiples.
+ * @returns {number} bytes
+ */
+export function parseDockerSize(value) {
+  const match = /^\s*([\d.]+)\s*([a-z]*)/i.exec(String(value ?? ''));
+  if (!match) {
+    return 0;
+  }
+  const unit = DECIMAL_UNITS[(match[2] || 'b').toLowerCase()] ?? 1;
+  return Math.round(Number(match[1]) * unit);
+}
+
+/**
+ * Parses newline-delimited JSON (`--format '{{json .}}'`).
+ * @returns {object[]}
+ */
+export function jsonLines(text) {
+  return (text ?? '')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith('{'))
+    .map((line) => JSON.parse(line));
+}
+
+/**
+ * Parses the comma-separated `Labels` column of `docker ps`.
+ * @returns {Record<string, string>}
+ */
+export function parseLabels(text) {
+  const labels = {};
+  for (const pair of (text ?? '').split(',')) {
+    const index = pair.indexOf('=');
+    if (index > 0) {
+      labels[pair.slice(0, index)] = pair.slice(index + 1);
+    }
+  }
+  return labels;
+}
+
+/**
+ * Paths of Git directories changed in a container's writable layer, from
+ * `docker diff` output, mapped to their work trees.
+ * @returns {string[]}
+ */
+export function repoRootsFromDiff(text) {
+  const roots = new Set();
+  for (const line of (text ?? '').split('\n')) {
+    const match = /^[ACD] (.+)$/.exec(line.trim());
+    if (!match) {
+      continue;
+    }
+    const gitIndex = match[1].search(/\/\.git(\/|$)/);
+    if (gitIndex !== -1) {
+      roots.add(match[1].slice(0, gitIndex) || '/');
+    }
+  }
+  return [...roots].sort();
+}
+
+export class DockerCli {
+  /**
+   * @param {object} executor where the docker CLI runs
+   * @param {{docker?: string}} [options]
+   */
+  constructor(executor, options = {}) {
+    this.executor = executor;
+    this.docker = options.docker ?? 'docker';
+  }
+
+  argv(args) {
+    assertAllowed(args);
+    return [this.docker, ...args];
+  }
+
+  run(args, options = {}) {
+    return this.executor.run(this.argv(args), options);
+  }
+
+  async json(args) {
+    const result = await this.run(args);
+    if (result.code !== 0) {
+      throw new Error(
+        `docker ${args.join(' ')} failed: ${result.stderr.trim() || result.code}`
+      );
+    }
+    return result.stdout;
+  }
+
+  /**
+   * Daemon facts, or null when no daemon is reachable.
+   * @returns {Promise<object|null>}
+   */
+  async info() {
+    const result = await this.run(['info', '--format', '{{json .}}'], {
+      timeoutMs: 30000,
+    });
+    if (result.code !== 0) {
+      return null;
+    }
+    try {
+      const info = JSON.parse(result.stdout);
+      return info.ID || info.ServerVersion ? info : null;
+    } catch {
+      return null;
+    }
+  }
+
+  async systemDf() {
+    return jsonLines(
+      await this.json(['system', 'df', '--format', '{{json .}}'])
+    );
+  }
+
+  async systemDfVerbose() {
+    return JSON.parse(
+      await this.json(['system', 'df', '-v', '--format', '{{json .}}'])
+    );
+  }
+
+  async containers() {
+    return jsonLines(
+      await this.json([
+        'ps',
+        '--all',
+        '--size',
+        '--no-trunc',
+        '--format',
+        '{{json .}}',
+      ])
+    );
+  }
+
+  async inspect(ids) {
+    if (ids.length === 0) {
+      return [];
+    }
+    try {
+      return JSON.parse(
+        await this.json(['inspect', '--type', 'container', ...ids])
+      );
+    } catch (error) {
+      if (ids.length === 1) {
+        throw error;
+      }
+      const found = [];
+      for (const id of ids) {
+        found.push(...(await this.inspect([id]).catch(() => [])));
+      }
+      return found;
+    }
+  }
+
+  async diff(id) {
+    const result = await this.run(['diff', id]);
+    return result.code === 0 ? result.stdout : null;
+  }
+
+  /** Check a small file in a stopped container without starting it. */
+  async pathExists(id, target) {
+    const result = await this.run(['cp', `${id}:${target}`, '-']);
+    if (result.code === 0) {
+      return true;
+    }
+    if (/could not find the file|no such file|not found/i.test(result.stderr)) {
+      return false;
+    }
+    return null;
+  }
+
+  async history(image) {
+    return jsonLines(
+      await this.json([
+        'history',
+        '--no-trunc',
+        '--format',
+        '{{json .}}',
+        image,
+      ])
+    );
+  }
+
+  /**
+   * Streams `docker cp <id>:<path> -` into `tar -x` on the machine running
+   * dss, skipping `excludes`. Works through nested executors because only
+   * the tar stream crosses them.
+   */
+  copyOut(id, source, destination, excludes = []) {
+    const stream = this.executor.spawn(
+      this.argv(['cp', `${id}:${source}`, '-'])
+    );
+    return pipeInto(stream, [
+      'tar',
+      '-x',
+      '-C',
+      destination,
+      ...excludes.map((pattern) => `--exclude=${pattern}`),
+    ]);
+  }
+}

package/src/docker/containers.js

@@ -0,0 +1,327 @@
+/**
+ * Stopped container inspection without starting the container.
+ *
+ * `docker rm` (without `-v`) deletes only the container's writable layer,
+ * so only work written there can be lost. `docker diff` lists that layer;
+ * every changed `.git` directory marks a repository, which is copied out
+ * with `docker cp` (dependency directories skipped) and checked with the
+ * host's Git.
+ */
+
+import { promises as fsp } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { GitInspector, stateBlockers } from '../git.js';
+import { LocalEnv } from '../env/local.js';
+import { repoRootsFromDiff } from './cli.js';
+
+export const COPY_EXCLUDES = [
+  '*/node_modules',
+  '*/target',
+  '*/.venv',
+  '*/venv',
+  '*/__pycache__',
+  '*/.gradle',
+  '*/.next',
+  '*/.nuxt',
+  '*/.tox',
+  '*/.mypy_cache',
+  '*/.pytest_cache',
+];
+
+const DEFAULT_MAX_REPOS = 20;
+const OWNER_HINT = /session|issue|pull|task|owner/i;
+const SECRET = /token|secret|password|key$/i;
+const TASK_URL_KEY = /(issue|pull|task)[\w.-]*url|url[\w.-]*(issue|pull|task)/i;
+const TASK_URL = /^https:\/\/[\w.-]+(\/[\w.~%#/-]*)?$/;
+const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
+const SHORT_ID = 12;
+const ENDED = new Set(['exited', 'dead']);
+
+function safeOwnerHint(key, value) {
+  const text = String(value);
+  return (
+    OWNER_HINT.test(key) &&
+    !SECRET.test(key) &&
+    !/https?:\/\/|ghp_|github_pat_|sk-/.test(text)
+  );
+}
+
+/** Only an exact container id can waive verified unsaved Git work. */
+export function gitBlockersForRemoval(git, id, options = {}) {
+  if (!(options.allowDirtyContainers ?? []).includes(id)) {
+    return git.blockers;
+  }
+  return git.blockers.filter(
+    (reason) => !/^Git repository .+ has /.test(reason)
+  );
+}
+
+/** Bind mounts of running containers in the current Docker daemon. */
+export function isHostBindSource(source) {
+  return Boolean(
+    source && (path.posix.isAbsolute(source) || path.win32.isAbsolute(source))
+  );
+}
+
+export async function runningBindMounts(docker) {
+  const running = (await docker.containers()).filter(
+    (row) => row.State === 'running'
+  );
+  const inspected = await docker.inspect(running.map((row) => row.ID));
+  return inspected.flatMap((container) =>
+    (container.Mounts ?? [])
+      .filter(
+        (mount) => mount.Type === 'bind' && isHostBindSource(mount.Source)
+      )
+      .map((mount) => ({ source: mount.Source, containerId: container.Id }))
+  );
+}
+
+/**
+ * Who a container belongs to: labels and environment variables that look
+ * like session, issue or task identifiers (`HIVE_MIND_PARENT_SESSION_ID`
+ * first among them), else a session UUID in the container name; plus its
+ * command, task URL and how it ended.
+ * @param {object} ps row of `docker ps --format json`
+ * @param {object} [inspected] `docker inspect` object
+ */
+export function ownerOf(ps, inspected) {
+  const hints = ownerHints(inspected);
+  const name = String(ps.Names ?? '').split(',')[0];
+  const [sessionSource, session] = sessionOf(hints, name);
+  const state = ENDED.has(inspected?.State?.Status) ? inspected.State : null;
+  return {
+    name,
+    image: ps.Image,
+    command: ps.Command ? ps.Command.replace(/^"|"$/g, '') : null,
+    createdAt: ps.CreatedAt ?? null,
+    finishedAt: inspected?.State?.FinishedAt ?? null,
+    exitCode: typeof state?.ExitCode === 'number' ? state.ExitCode : null,
+    oomKilled: Boolean(state?.OOMKilled),
+    exitReason: exitReason(state),
+    taskUrl: taskUrlOf(inspected),
+    session,
+    sessionSource,
+    hints,
+  };
+}
+
+function sessionOf(hints, name) {
+  const entries = Object.entries(hints);
+  const found =
+    entries.find(([key]) => /HIVE_MIND_PARENT_SESSION_ID$/.test(key)) ??
+    entries.find(([key]) => /session/i.test(key)) ??
+    entries[0];
+  if (found) {
+    return found;
+  }
+  const fromName = UUID.exec(name)?.[0];
+  return fromName ? ['name', fromName] : [null, null];
+}
+
+/**
+ * How a stopped container ended, from `docker inspect` State.
+ * @returns {string|null}
+ */
+export function exitReason(state) {
+  if (!ENDED.has(state?.Status) || typeof state.ExitCode !== 'number') {
+    return null;
+  }
+  if (state.Error) {
+    return state.Error;
+  }
+  if (state.OOMKilled) {
+    return 'killed: out of memory';
+  }
+  if (state.ExitCode === 0) {
+    return 'completed';
+  }
+  return state.ExitCode > 128
+    ? `killed by signal ${state.ExitCode - 128}`
+    : `failed with exit code ${state.ExitCode}`;
+}
+
+/**
+ * Why a task runner keeps a stopped container for investigation (it failed
+ * or was OOM killed), or null.
+ */
+export function investigationReason(state) {
+  const parts = [];
+  if (state?.ExitCode) {
+    parts.push(`exit ${state.ExitCode}`);
+  }
+  if (state?.OOMKilled) {
+    parts.push('OOM killed');
+  }
+  return parts.length > 0
+    ? `kept for investigation (${parts.join(', ')})`
+    : null;
+}
+
+/**
+ * Whether the operator named this container with `--remove-container`:
+ * its name, full id or an id prefix of at least 12 characters.
+ */
+export function containerNamed(names, id, name) {
+  return (names ?? []).some(
+    (entry) =>
+      entry === name ||
+      entry === `/${name}` ||
+      (entry.length >= SHORT_ID && id.startsWith(entry))
+  );
+}
+
+/** Blocker of a container kept for investigation that was not named. */
+export function investigationBlocker(state, id, name, options = {}) {
+  const reason = investigationReason(state);
+  if (!reason || containerNamed(options.removeContainers, id, name)) {
+    return null;
+  }
+  return `${reason}; remove it only with --remove-container ${id.slice(0, SHORT_ID)}`;
+}
+
+function taskUrlOf(inspected) {
+  const pairs = [
+    ...Object.entries(inspected?.Config?.Labels ?? {}),
+    ...(inspected?.Config?.Env ?? []).map((pair) => {
+      const index = pair.indexOf('=');
+      return [pair.slice(0, index), pair.slice(index + 1)];
+    }),
+  ];
+  const found = pairs.find(
+    ([key, value]) =>
+      TASK_URL_KEY.test(key) &&
+      !SECRET.test(key) &&
+      TASK_URL.test(value) &&
+      !/ghp_|github_pat_|sk-/.test(value)
+  );
+  return found?.[1] ?? null;
+}
+
+function ownerHints(inspected) {
+  const hints = {};
+  for (const [key, value] of Object.entries(inspected?.Config?.Labels ?? {})) {
+    if (safeOwnerHint(key, value)) {
+      hints[`label:${key}`] = value;
+    }
+  }
+  for (const pair of inspected?.Config?.Env ?? []) {
+    const index = pair.indexOf('=');
+    const key = pair.slice(0, index);
+    if (index > 0 && safeOwnerHint(key, pair.slice(index + 1))) {
+      hints[`env:${key}`] = pair.slice(index + 1);
+    }
+  }
+  return hints;
+}
+
+async function copyAndCheck(docker, id, root, git) {
+  const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'dss-container-git-'));
+  try {
+    const copied = await docker.copyOut(id, root, dir, COPY_EXCLUDES);
+    if (copied.code !== 0) {
+      return {
+        root,
+        error: `docker cp failed: ${copied.stderr.trim() || copied.code}`,
+      };
+    }
+    const local = path.join(dir, path.basename(root));
+    const state = await git.repoState(local);
+    return { ...state, root };
+  } finally {
+    await fsp.rm(dir, { recursive: true, force: true });
+  }
+}
+
+/**
+ * Git state of every repository written in a stopped container.
+ * @returns {Promise<{repos: object[], blockers: string[]}>}
+ */
+async function probeRepoRoots(docker, id, changed, roots, blockers) {
+  if (changed.length > 500) {
+    blockers.push('Git repository inspection limit exceeded');
+  }
+  const probed = new Map();
+  for (const target of changed.slice(0, 500)) {
+    for (let dir = target; dir !== '/'; dir = path.posix.dirname(dir)) {
+      if (probed.has(dir)) {
+        if (probed.get(dir)) {
+          roots.add(dir);
+        }
+        break;
+      }
+      const exists = await docker.pathExists(
+        id,
+        path.posix.join(dir, '.git', 'HEAD')
+      );
+      if (exists === null) {
+        blockers.push(`cannot inspect Git metadata near ${target}`);
+        break;
+      }
+      probed.set(dir, exists);
+      if (exists) {
+        roots.add(dir);
+        break;
+      }
+    }
+  }
+}
+
+function excludedWorkBlockers(checked, changed, blockers) {
+  const excluded = new Set(COPY_EXCLUDES.map((pattern) => pattern.slice(2)));
+  for (const root of checked) {
+    for (const target of changed) {
+      if (!target.startsWith(`${root}/`)) {
+        continue;
+      }
+      const parts = target.slice(root.length + 1).split('/');
+      if (parts.slice(0, -1).some((part) => excluded.has(part))) {
+        blockers.push(
+          `changed work under excluded folder ${target} cannot be verified`
+        );
+      }
+    }
+  }
+}
+
+export async function containerGitState(docker, id, options = {}) {
+  const diff = await docker.diff(id);
+  if (diff === null) {
+    return {
+      repos: [],
+      blockers: ['cannot list the writable layer (`docker diff` failed)'],
+    };
+  }
+  const roots = new Set(repoRootsFromDiff(diff));
+  const maxRepos = options.maxRepos ?? DEFAULT_MAX_REPOS;
+  const blockers = [];
+  if (roots.has('/')) {
+    blockers.push('the container root is a Git repository');
+  }
+  const changed = diff
+    .split('\n')
+    .map((line) => /^[ACD] (\/.+)$/.exec(line.trim())?.[1])
+    .filter(Boolean);
+  await probeRepoRoots(docker, id, changed, roots, blockers);
+  const checked = [...roots].filter((root) => root !== '/').slice(0, maxRepos);
+  excludedWorkBlockers(checked, changed, blockers);
+  if (roots.size > checked.length + (roots.has('/') ? 1 : 0)) {
+    blockers.push(
+      `${roots.size} repositories changed, only ${maxRepos} were verified`
+    );
+  }
+  const repos = [];
+  if (checked.length === 0) {
+    return { repos, blockers };
+  }
+  // `docker cp` copies to this machine, so Git always runs locally.
+  const git = new GitInspector(options.env ?? new LocalEnv());
+  for (const root of checked) {
+    const state = await copyAndCheck(docker, id, root, git);
+    repos.push(state);
+    blockers.push(...stateBlockers(`${root} (in container)`, state));
+  }
+  return { repos, blockers };
+}