disk-space-saviour 0.20.0 → 0.22.0

package/src/env/local.js

@@ -89,7 +89,7 @@ export function parsePsSearchPaths(output) {
  * host and inside containers (see shell.js).
  */
 
-import { promises as fsp } from 'node:fs';
+import { constants as fsConstants, promises as fsp } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { URL, fileURLToPath } from 'node:url';
@@ -97,6 +97,7 @@ import { procIdentity, procOpenPaths } from './proc.js';
 import { CACHE_VARIABLES } from './cache-roots.js';
 
 import { hostExecutor } from '../exec.js';
+import { parseBundleIdentifier } from '../app-owners.js';
 import { matchesGlob } from '../paths.js';
 import { mapLimit } from './parallel.js';
 import { directoryFingerprint, SIZE_MAX_AGE } from '../scan-state.js';
@@ -251,6 +252,36 @@ export class LocalEnv {
     );
   }
 
+  /**
+   * Access errors as report messages. Paths inside a directory this user
+   * cannot enter collapse into one line: nothing inside such a directory
+   * can be deleted by this user, so no candidate hides there.
+   * @returns {Promise<string[]>}
+   */
+  async accessErrorMessages() {
+    const messages = [];
+    const foreign = new Set();
+    for (const [target, message] of this.accessErrors ?? []) {
+      const parent = path.dirname(target);
+      const enterable = await fsp
+        .access(parent, fsConstants.X_OK)
+        .then(() => true)
+        .catch(() => false);
+      if (enterable) {
+        messages.push(message);
+      } else {
+        foreign.add(parent);
+      }
+    }
+    if (foreign.size > 0) {
+      const [example] = [...foreign].sort();
+      messages.push(
+        `skipped ${foreign.size} ${foreign.size === 1 ? 'directory' : 'directories'} this user cannot enter, such as ${example}; nothing inside can be deleted without their owner's rights`
+      );
+    }
+    return messages;
+  }
+
   async #lstat(target) {
     try {
       return await (this.options.fileFs ?? fsp).lstat(target);
@@ -333,6 +364,24 @@ export class LocalEnv {
     return heads;
   }
 
+  /** Another user's entry of a sticky shared directory such as /tmp is private by design. */
+  async #foreignTempEntry(dir) {
+    const io = this.options.fileFs ?? fsp;
+    try {
+      const [own, parent] = await Promise.all([
+        io.lstat(dir),
+        io.stat(path.dirname(dir)),
+      ]);
+      return (
+        typeof process.getuid === 'function' &&
+        own.uid !== process.getuid() &&
+        (parent.mode & 0o1000) !== 0
+      );
+    } catch {
+      return false;
+    }
+  }
+
   async list(dir, { discovery = false } = {}) {
     let names;
     try {
@@ -340,7 +389,9 @@ export class LocalEnv {
         withFileTypes: true,
       });
     } catch (error) {
-      this.recordAccessError(dir, error);
+      if (!(await this.#foreignTempEntry(dir))) {
+        this.recordAccessError(dir, error);
+      }
       return [];
     }
     const entries = await mapLimit(names, 32, async (entry) => {
@@ -810,6 +861,8 @@ export class LocalEnv {
       .map((m) => ({
         pid: Number(m[1]),
         name: path.basename(m[2].trim()),
+        // macOS `comm` is the executable's full path.
+        executable: m[2].trim(),
         cwd: cwds.get(Number(m[1])) ?? null,
         command: commands.get(Number(m[1])) ?? '',
         searchPath: searchPaths.get(Number(m[1])) ?? [],
@@ -958,12 +1011,22 @@ export class LocalEnv {
       if (result.code !== 0 && !result.stdout) {
         return null;
       }
-      return new Set(
-        result.stdout
-          .split('\n')
-          .filter((line) => line.startsWith('n/'))
-          .map((line) => line.slice(1))
-      );
+      // `-F` always prints the pid (`p`) before the files of a process.
+      const paths = new Set();
+      const owners = new Map();
+      let pid = null;
+      for (const line of result.stdout.split('\n')) {
+        if (line.startsWith('p')) {
+          pid = Number(line.slice(1));
+        } else if (line.startsWith('n/')) {
+          paths.add(line.slice(1));
+          if (!owners.has(line.slice(1))) {
+            owners.set(line.slice(1), pid);
+          }
+        }
+      }
+      this.openPathPids = owners;
+      return paths;
     }
     return null;
   }
@@ -972,6 +1035,26 @@ export class LocalEnv {
     return this.executor.run(argv, options);
   }
 
+  /** `CFBundleIdentifier` of a macOS bundle, or null. */
+  async bundleIdentifier(bundle) {
+    if (this.platform !== 'darwin') {
+      return null;
+    }
+    const plist = path.join(bundle, 'Contents', 'Info.plist');
+    const text = await fsp.readFile(plist, 'utf8').catch(() => null);
+    if (text === null) {
+      return null;
+    }
+    if (!text.startsWith('bplist')) {
+      return parseBundleIdentifier(text);
+    }
+    const result = await this.run(
+      ['plutil', '-extract', 'CFBundleIdentifier', 'raw', '-o', '-', plist],
+      { timeoutMs: 10000 }
+    ).catch(() => null);
+    return result?.code === 0 ? result.stdout.trim() || null : null;
+  }
+
   async which(command) {
     const dirs = (process.env.PATH ?? '').split(path.delimiter);
     const extensions =

package/src/git-remote.js

@@ -0,0 +1,168 @@
+/**
+ * Proof that a repository's commits still exist on a network remote, and
+ * the linked worktrees that share its object database.
+ *
+ * Remote URLs are read from the repository config but never used through
+ * it: `git ls-remote` runs outside any repository, with only the https and
+ * ssh transports allowed and no prompts, so a planted config cannot run
+ * commands or ask for credentials. Only advertised branches, tags and
+ * review refs (`refs/pull/*`, `refs/merge-requests/*`) count, and only the
+ * advertised commits that exist locally, so "on the remote" never relies
+ * on cached remote-tracking refs.
+ */
+import { promises as fsp } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const LS_REMOTE_TIMEOUT_MS = 30 * 1000;
+const ADVERTISED = [
+  'refs/heads/*',
+  'refs/tags/*',
+  'refs/pull/*/head',
+  'refs/merge-requests/*/head',
+];
+
+/**
+ * A remote URL `git ls-remote` may contact: https/http or ssh, with a host
+ * that cannot be mistaken for an option. Local paths and `file://` remotes
+ * are not proof that the code exists elsewhere.
+ */
+export function networkRemote(url) {
+  const value = url.trim();
+  if (/\s/.test(value) || value.startsWith('-')) {
+    return null;
+  }
+  const standard = /^(https?|ssh|git\+ssh):\/\/(?:[^@/]+@)?([^/:]+)/i.exec(
+    value
+  );
+  const scp = /^(?:[\w.-]+@)?([\w.-]+):(?!\/\/)[^\\]/.exec(value);
+  const host = standard?.[2] ?? scp?.[1];
+  // `C:/repo` is a Windows path, not the host `C`.
+  const drive = !standard && /^[a-z]$/i.test(host ?? '');
+  return host && !drive && /^[\w.[\]-]+$/.test(host) && !host.startsWith('-')
+    ? { url: value, host }
+    : null;
+}
+
+/** `remote.<name>.url` values from a Git config, as `{name, url}`. */
+export function parseRemoteUrls(text) {
+  const remotes = [];
+  let remote = null;
+  for (const raw of text.split('\n')) {
+    const line = raw.trim();
+    const section = /^\[\s*remote\s+"([^"]+)"\s*\]$/i.exec(line);
+    if (section) {
+      remote = section[1];
+      continue;
+    }
+    if (/^\[/.test(line)) {
+      remote = null;
+      continue;
+    }
+    const url = /^url\s*=\s*(.+)$/i.exec(line);
+    if (remote && url) {
+      remotes.push({ name: remote, url: url[1].replace(/^"(.*)"$/, '$1') });
+    }
+  }
+  return remotes;
+}
+
+/** Commit IDs from `git ls-remote` output, peeled tags included. */
+export function advertisedCommits(text) {
+  return [
+    ...new Set(
+      text
+        .split('\n')
+        .map((line) => /^([0-9a-f]{40,64})\t/.exec(line)?.[1])
+        .filter(Boolean)
+    ),
+  ];
+}
+
+/** Network remotes of the repository whose common Git directory is given. */
+export async function networkRemotes(commonDir) {
+  const text = await fsp
+    .readFile(path.join(commonDir, 'config'), 'utf8')
+    .catch(() => '');
+  return parseRemoteUrls(text)
+    .map((remote) => ({ name: remote.name, ...networkRemote(remote.url) }))
+    .filter((remote) => remote.host);
+}
+
+/**
+ * Work trees attached to a common Git directory: the main one (unless the
+ * repository is bare) and every linked one that still exists, with the
+ * detached HEAD commits of linked worktrees.
+ */
+export async function attachedWorktrees(commonDir) {
+  const trees = [];
+  const detached = [];
+  const config = await fsp
+    .readFile(path.join(commonDir, 'config'), 'utf8')
+    .catch(() => '');
+  if (
+    path.basename(commonDir) === '.git' &&
+    !/^\s*bare\s*=\s*true/m.test(config)
+  ) {
+    trees.push(path.dirname(commonDir));
+  }
+  const base = path.join(commonDir, 'worktrees');
+  for (const name of await fsp.readdir(base).catch(() => [])) {
+    const read = (file) =>
+      fsp.readFile(path.join(base, name, file), 'utf8').catch(() => '');
+    const gitdir = (await read('gitdir')).trim();
+    const head = (await read('HEAD')).trim();
+    if (/^[0-9a-f]{40,64}$/.test(head)) {
+      detached.push(head);
+    }
+    if (gitdir) {
+      const tree = path.dirname(path.resolve(base, name, gitdir));
+      if (await fsp.stat(tree).catch(() => null)) {
+        trees.push(tree);
+      }
+    }
+  }
+  return { trees, detached };
+}
+
+/**
+ * Runs `git ls-remote` for `url` outside every repository.
+ * @param {(argv: string[], options: object) => Promise<object>} run
+ */
+export async function lsRemote(run, url) {
+  const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'dss-ls-remote-'));
+  try {
+    return await run(
+      [
+        'git',
+        '-c',
+        'protocol.allow=never',
+        '-c',
+        'protocol.https.allow=always',
+        '-c',
+        'protocol.http.allow=always',
+        '-c',
+        'protocol.ssh.allow=always',
+        '-c',
+        'core.hooksPath=/dev/null',
+        '-C',
+        dir,
+        'ls-remote',
+        url,
+        ...ADVERTISED,
+      ],
+      {
+        timeoutMs: LS_REMOTE_TIMEOUT_MS,
+        env: {
+          GIT_TERMINAL_PROMPT: '0',
+          GIT_ASKPASS: '',
+          SSH_ASKPASS: '',
+          GIT_SSH_COMMAND: 'ssh -o BatchMode=yes -o ConnectTimeout=15',
+          GIT_CEILING_DIRECTORIES: path.dirname(dir),
+        },
+      }
+    );
+  } finally {
+    await fsp.rm(dir, { recursive: true, force: true });
+  }
+}

package/src/git.js

@@ -16,6 +16,12 @@ import path from 'node:path';
 
 import { trace } from './exec.js';
 import { preservedWork } from './git-preserved.js';
+import {
+  advertisedCommits,
+  attachedWorktrees,
+  lsRemote,
+  networkRemotes,
+} from './git-remote.js';
 
 const GIT_FLAGS = [
   '--no-optional-locks',
@@ -42,7 +48,7 @@ const SAFE_GIT_ENV = {
  * a linked worktree or a submodule has a `.git` file naming its directory,
  * and a linked worktree names the common directory in `commondir`.
  */
-async function gitDirectories(root) {
+export async function gitDirectories(root) {
   const dotGit = path.join(root, '.git');
   const stat = await fsp.lstat(dotGit);
   let gitDir = dotGit;
@@ -176,10 +182,11 @@ export class GitInspector {
     this.options = options;
     this.states = new Map();
     this.roots = new Map();
+    this.remoteRefs = new Map();
     this.available = null;
   }
 
-  async #git(root, args, safeDir = null) {
+  async #git(root, args, safeDir = null, input = undefined) {
     const result = await this.env.run(
       [
         'git',
@@ -189,7 +196,7 @@ export class GitInspector {
         ...(safeDir ? ['--git-dir', safeDir, '--work-tree', root] : []),
         ...args,
       ],
-      { env: SAFE_GIT_ENV }
+      { env: SAFE_GIT_ENV, input }
     );
     trace('git', root, args.join(' '), '->', result.code);
     return result;
@@ -246,6 +253,7 @@ export class GitInspector {
   invalidate() {
     this.states.clear();
     this.roots.clear();
+    this.remoteRefs.clear();
   }
 
   async #loadState(root) {
@@ -322,8 +330,8 @@ export class GitInspector {
     return stateBlockers(root, await this.repoState(root));
   }
 
-  /** Protect an entire temporary work directory, including nested clones. */
-  async treeBlockers(target, { allowNoRepo = false } = {}) {
+  /** Repositories at or below `target`. */
+  async #repoRoots(target) {
     const dirs = await this.env.findDirs([target], {
       globs: ['.git'],
       maxDepth: 32,
@@ -334,13 +342,18 @@ export class GitInspector {
       maxDepth: 32,
       skipNames: [],
     });
-    const roots = new Set([
+    return new Set([
       ...((await this.env.exists(this.env.path.join(target, '.git')))
         ? [target]
         : []),
       ...dirs.map((entry) => entry.parent),
       ...files.map((file) => this.env.path.dirname(file)),
     ]);
+  }
+
+  /** Protect an entire temporary work directory, including nested clones. */
+  async treeBlockers(target, { allowNoRepo = false } = {}) {
+    const roots = await this.#repoRoots(target);
     if (roots.size === 0) {
       if (allowNoRepo) {
         return [];
@@ -357,6 +370,134 @@ export class GitInspector {
     return blockers;
   }
 
+  /**
+   * Reasons a temporary clone must not be deleted. Every repository inside
+   * `target` must have no uncommitted, unpushed or stashed work in any of
+   * its worktrees, no linked worktree outside `target`, and every local
+   * commit must be advertised by one of its https or ssh remotes.
+   */
+  async cloneBlockers(target) {
+    if (this.env.kind !== 'host') {
+      return [`${target}: remote copies are only verified on the host`];
+    }
+    if (!(await this.isAvailable())) {
+      return [`${target}: git is not installed, cannot verify the repository`];
+    }
+    const roots = await this.#repoRoots(target);
+    if (roots.size === 0) {
+      return [`${target}: no Git repository found; cannot verify work`];
+    }
+    if (roots.size > 50) {
+      return [`${target}: too many repositories to verify safely`];
+    }
+    const blockers = [];
+    for (const root of roots) {
+      blockers.push(...(await this.#cloneRootBlockers(root, target)));
+    }
+    return blockers;
+  }
+
+  async #cloneRootBlockers(root, target) {
+    let commonDir;
+    try {
+      ({ commonDir } = await gitDirectories(root));
+    } catch (error) {
+      return [`${root}: ${error.message}`];
+    }
+    const inside = (dir) => {
+      const relative = path.relative(target, dir);
+      return (
+        relative === '' ||
+        (!relative.startsWith('..') && !path.isAbsolute(relative))
+      );
+    };
+    const { trees, detached } = await attachedWorktrees(commonDir);
+    // Deleting the repository strands every worktree that shares it.
+    const affected = inside(commonDir) ? [root, ...trees] : [root];
+    const blockers = [];
+    for (const tree of new Set(affected)) {
+      if (!inside(tree)) {
+        blockers.push(`linked worktree ${tree} uses the repository in ${root}`);
+      } else if (await this.env.exists(path.join(tree, '.git'))) {
+        blockers.push(...(await this.repoBlockers(tree)));
+      }
+    }
+    if (blockers.length > 0) {
+      return blockers;
+    }
+    return this.#remoteBlockers(root, commonDir, detached);
+  }
+
+  async #remoteBlockers(root, commonDir, detached) {
+    const remotes = await networkRemotes(commonDir);
+    if (remotes.length === 0) {
+      return [`Git repository ${root} has no https or ssh remote holding it`];
+    }
+    const advertised = [];
+    const unreachable = [];
+    for (const remote of remotes) {
+      if (!this.remoteRefs.has(remote.url)) {
+        this.remoteRefs.set(
+          remote.url,
+          lsRemote((argv, options) => this.env.run(argv, options), remote.url)
+        );
+      }
+      const result = await this.remoteRefs.get(remote.url);
+      if (result.code === 0) {
+        advertised.push(...advertisedCommits(result.stdout));
+      } else {
+        unreachable.push(`${remote.name} (${remote.host})`);
+      }
+    }
+    if (advertised.length === 0) {
+      return [
+        `cannot confirm Git repository ${root} on a remote: ${unreachable.length > 0 ? `${unreachable.join(', ')} unreachable` : 'its remotes advertise nothing'}`,
+      ];
+    }
+    try {
+      const missing = await safeGitDirectory(root, (safeDir) =>
+        this.#commitsMissingFrom(root, safeDir, advertised, detached)
+      );
+      if (missing === 0) {
+        return [];
+      }
+      const hosts = [...new Set(remotes.map((remote) => remote.host))];
+      return [
+        `Git repository ${root} has ${plural(missing, 'commit')} no branch, tag or pull request on ${hosts.join(', ')} contains`,
+      ];
+    } catch (error) {
+      return [`cannot compare ${root} with its remote: ${error.message}`];
+    }
+  }
+
+  /** Local commits reachable from HEAD, branches or `detached` but no advertised commit. */
+  async #commitsMissingFrom(root, safeDir, advertised, detached) {
+    const known = await this.#git(
+      root,
+      ['cat-file', '--batch-check=%(objectname) %(objecttype)'],
+      safeDir,
+      `${advertised.join('\n')}\n`
+    );
+    if (known.code !== 0) {
+      throw new Error(known.stderr.trim() || `exit ${known.code}`);
+    }
+    const commits = known.stdout
+      .split('\n')
+      .map((line) => /^([0-9a-f]+) commit$/.exec(line)?.[1])
+      .filter(Boolean);
+    const tips = ['HEAD', '--branches', ...detached];
+    const count = await this.#git(
+      root,
+      ['rev-list', '--count', '--stdin', ...tips],
+      safeDir,
+      commits.map((commit) => `^${commit}\n`).join('')
+    );
+    if (count.code !== 0) {
+      throw new Error(count.stderr.trim() || `exit ${count.code}`);
+    }
+    return Number(count.stdout.trim());
+  }
+
   /**
    * Number of tracked files inside `target` (a path within `root`).
    * Returns -1 when Git cannot answer.

package/src/index.d.ts

@@ -172,9 +172,19 @@ export interface EnvironmentDescriptor {
     holders: Array<{
       pid: number;
       command: string;
+      /** Resident plus compressed and swapped memory. */
       memoryBytes: number;
-      compressedBytes: number;
+      /** macOS only. */
+      compressedBytes?: number;
+      /** Linux only (`VmSwap`). */
+      swappedBytes?: number;
+      /** Change since the earliest scan of the last week listing this process. */
+      growth?: { since: number; memoryBytes: number };
     }>;
+    /** Change since the earliest scan of the last week. */
+    growth?: { since: number; usedBytes: number } | null;
+    /** Large, growing, or disk-threatening swap, naming the memory holders. */
+    warnings?: string[];
     attribution: string;
     advice: string;
   } | null;
@@ -209,6 +219,16 @@ export interface Report {
   options: Record<string, unknown>;
   environments: EnvironmentDescriptor[];
   items: Item[];
+  /**
+   * Host items that share every block with an installed app (APFS clones):
+   * never offered, listed so the space `du` counts for them is explained.
+   */
+  shared?: Array<{
+    rule: string;
+    path: string;
+    apparentBytes: number;
+    sharedWith: string;
+  }>;
   /** Tier totals exclude items that need explicit consent (`consent`). */
   totals: Record<Tier | 'blocked' | 'consent', TierTotal>;
   docker: {
@@ -290,6 +310,50 @@ export declare function clean(
 /** Escalates tiers until the free-space goal is met, then stops. */
 export declare function emergency(options: EmergencyOptions): Promise<AuditLog>;
 
+export interface UsageOptions {
+  /** Directories to walk; default the macOS Data volume or `/`. */
+  roots?: string[] | null;
+  /** Smallest directory in the tree, in bytes; default 1% of a root. */
+  minSize?: number;
+  verbose?: boolean;
+  signal?: AbortSignal;
+  auditDir?: string;
+  onUsageProgress?: (progress: {
+    root: string;
+    elapsedMs: number;
+    entries: number;
+    at: string | null;
+  }) => void;
+}
+
+export interface UsageReport {
+  schema: 1;
+  createdAt: string;
+  durationMs: number;
+  platform: string;
+  scan: { file: string; createdAt: string | null } | null;
+  roots: Array<{
+    root: string;
+    path: string;
+    bytes: number;
+    entries: number;
+    tree: object;
+    files: Array<{ path: string; bytes: number }>;
+    [key: string]: unknown;
+  }>;
+  volumes: Array<{
+    name: string;
+    mount: string | null;
+    bytes: number;
+    role: string;
+  }>;
+  snapshots: number | null;
+}
+
+/** Shows where the space of whole volumes goes. Never deletes. */
+export declare function usage(options?: UsageOptions): Promise<UsageReport>;
+export declare function formatUsage(report: UsageReport): string;
+
 export declare function formatReport(
   report: Report,
   options?: { verbose?: boolean }

package/src/index.js

@@ -23,6 +23,7 @@ export {
   writeAudit,
 } from './audit.js';
 export { formatReport, formatAudit, formatDisk } from './report.js';
+export { usage, formatUsage } from './usage.js';
 export { runCli, parseCli, USAGE } from './cli.js';
 export { DEFAULTS, resolveOptions, resolveGoal } from './options.js';
 export {

package/src/items.js

@@ -7,6 +7,8 @@
  * named flag (or an interactive yes) on top of their tier being selected.
  */
 
+import { blockedConditions } from './conditions.js';
+
 export const TIERS = ['safe', 'moderate', 'aggressive'];
 
 /**
@@ -111,6 +113,7 @@ export function tierTotals(items) {
     unknownBytes: blocked
       .filter((item) => item.sizeUnknown)
       .reduce((sum, item) => sum + (item.totalBytes ?? item.bytes), 0),
+    ...blockedConditions(blocked),
   };
   return totals;
 }