disk-space-saviour 0.15.2 → 0.17.0

package/src/cli.js

@@ -15,9 +15,11 @@ import { startAudit, writeAudit } from './audit.js';
 import { clean } from './clean.js';
 import { emergency } from './emergency.js';
 import { setTrace } from './exec.js';
-import { resolveGoal } from './options.js';
+import { resolveGoal, resolveOptions } from './options.js';
 import { formatAudit, formatReport } from './report.js';
 import { scan } from './scan.js';
+import { cleanDuringScan } from './fast-clean.js';
+import { defaultStateDirectory, loadScan } from './scan-state.js';
 import { formatBytes } from './units.js';
 
 export const USAGE = `Usage: dss <command> [paths...] [options]
@@ -37,6 +39,7 @@ Scope:
   --recursive                docker: recurse into nested daemons
   --container ID|NAME        limit Docker work to these containers (repeat)
   --max-depth N              directory depth for project search (default 6)
+  --scan-budget AGE          time budget per filesystem environment (2m)
   --only NAME                only these ecosystems, rules or kinds (repeat)
   --exclude PATH|GLOB        never touch matching paths (repeat)
   --scanner NAME             projects, global, versions, agents, system
@@ -66,6 +69,9 @@ Consent:
 Output:
   --json                     print the report or audit log as JSON
   --report FILE              clean: use a report saved by scan --json
+  --from FILE                clean/emergency: reuse a scan or audit JSON
+  --fresh                    bypass the recent scan and persistent index
+  --cache-dir DIR            scan state (default: XDG state directory)
   --audit-dir DIR            audit logs ($DSS_AUDIT_DIR)
   --backup-dir DIR           container log backups ($DSS_BACKUP_DIR)
   --verbose                  list everything and trace commands (DSS_DEBUG=1)
@@ -80,6 +86,7 @@ const OPTIONS = {
   recursive: { type: 'boolean' },
   container: { type: 'string', multiple: true },
   'max-depth': { type: 'string' },
+  'scan-budget': { type: 'string' },
   only: { type: 'string', multiple: true },
   exclude: { type: 'string', multiple: true },
   scanner: { type: 'string', multiple: true },
@@ -104,6 +111,9 @@ const OPTIONS = {
   'allow-dirty-container': { type: 'string', multiple: true },
   'no-native': { type: 'boolean' },
   report: { type: 'string' },
+  from: { type: 'string' },
+  fresh: { type: 'boolean' },
+  'cache-dir': { type: 'string' },
   'audit-dir': { type: 'string' },
   'backup-dir': { type: 'string' },
   help: { type: 'boolean', short: 'h' },
@@ -158,6 +168,7 @@ export function toOptions(values, paths = []) {
     dockerDepth: integer(values.depth, '--depth'),
     containers: values.container,
     maxDepth: integer(values['max-depth'], '--max-depth'),
+    scanBudget: values['scan-budget'],
     only: values.only,
     exclude: values.exclude,
     scanners: values.scanner,
@@ -179,6 +190,9 @@ export function toOptions(values, paths = []) {
     noNative: values['no-native'],
     auditDir: values['audit-dir'],
     backupDir: values['backup-dir'],
+    cacheDir: values.fresh
+      ? false
+      : (values['cache-dir'] ?? defaultStateDirectory()),
   };
 }
 
@@ -237,14 +251,7 @@ function dockerOnly(options, values) {
 export function scanAudit(report) {
   return startAudit('scan', {
     dryRun: true,
-    report: {
-      createdAt: report.createdAt,
-      durationMs: report.durationMs,
-      totals: report.totals,
-      environments: report.environments.map((env) => env.id),
-      errors: report.errors,
-      items: report.items,
-    },
+    report,
     entries: report.items.map((item) => ({
       id: item.id,
       env: item.env,
@@ -280,7 +287,7 @@ export function processIo() {
   };
 }
 
-const API = { scan, clean, emergency };
+const API = { scan, clean, emergency, cleanDuringScan };
 
 class Cli {
   constructor(io, api) {
@@ -294,9 +301,18 @@ class Cli {
 
   async scan(options, values) {
     const report = await this.api.scan(options);
+    return this.printScan(report, options, values);
+  }
+
+  async printScan(report, options, values) {
     const audit = scanAudit(report);
     if (options.audit !== false) {
-      await writeAudit(audit, options);
+      try {
+        await writeAudit(audit, options);
+      } catch (error) {
+        report.audit = { file: null, error: error.message, code: error.code };
+        this.io.stderr(`Audit log: not written (${error.message})`);
+      }
     }
     this.print(values, report, () =>
       [
@@ -305,14 +321,23 @@ class Cli {
         'Run `dss clean --tier safe` to see the plan, add --yes to delete.',
       ].join('\n')
     );
-    return report.errors.length > 0 ? 1 : 0;
+    return report.aborted ? 130 : report.errors.length > 0 ? 1 : 0;
   }
 
   async loadReport(options, values) {
-    if (!values.report) {
+    const file = values.from ?? values.report;
+    if (!file) {
+      const recent = await loadScan(resolveOptions(options));
+      if (recent) {
+        this.io.stderr(
+          `Reusing scan from ${recent.createdAt}; planned items will be checked before deletion.`
+        );
+        return recent;
+      }
       return this.api.scan(options);
     }
-    return JSON.parse(await fsp.readFile(values.report, 'utf8'));
+    const saved = JSON.parse(await fsp.readFile(file, 'utf8'));
+    return saved.report?.schema ? saved.report : saved;
   }
 
   confirmItem() {
@@ -359,7 +384,37 @@ class Cli {
   }
 
   async clean(options, values) {
+    if (
+      this.api.cleanDuringScan &&
+      !values.report &&
+      !values.from &&
+      !this.io.interactive
+    ) {
+      const recent = await loadScan(resolveOptions(options));
+      if (!recent) {
+        const result = await this.consent(values, (extra) =>
+          this.api.cleanDuringScan({
+            ...options,
+            ...extra,
+            streaming: options.docker !== false,
+          })
+        );
+        if (result.report.aborted) {
+          return this.printScan(result.report, options, values);
+        }
+        this.print(values, result.audit, () =>
+          formatAudit(result.audit, { verbose: values.verbose })
+        );
+        return result.audit.aborted ||
+          result.audit.entries.some((entry) => entry.status === 'failed')
+          ? 1
+          : 0;
+      }
+    }
     const report = await this.loadReport(options, values);
+    if (report.aborted) {
+      return this.printScan(report, options, values);
+    }
     const audit = await this.consent(values, (extra) =>
       this.api.clean(report, { ...options, ...extra })
     );
@@ -378,7 +433,36 @@ class Cli {
     } catch (error) {
       throw new UsageError(error.message);
     }
+    if (
+      this.api === API &&
+      !values.report &&
+      !values.from &&
+      !this.io.interactive
+    ) {
+      const recent = await loadScan(resolveOptions(options));
+      const audit = await this.consent(values, (extra) =>
+        this.api.emergency({
+          ...options,
+          ...(recent ? { report: recent } : {}),
+          ...extra,
+        })
+      );
+      if (audit.scanReport?.aborted) {
+        return this.printScan(audit.scanReport, options, values);
+      }
+      this.print(values, audit, () =>
+        formatAudit(audit, { verbose: values.verbose })
+      );
+      return audit.entries.some((entry) => entry.status === 'failed')
+        ? 1
+        : audit.goalMet
+          ? 0
+          : 3;
+    }
     const report = await this.loadReport(options, values);
+    if (report.aborted) {
+      return this.printScan(report, options, values);
+    }
     const audit = await this.consent(values, (extra) =>
       this.api.emergency({ ...options, report, ...extra })
     );
@@ -441,6 +525,10 @@ export async function runCli(argv, deps = {}) {
     if (deps.signal) {
       options.signal = deps.signal;
     }
+    options.onScanProgress = (progress) =>
+      io.stderr(
+        `[scan] ${progress.env}: ${progress.scanner}, ${Math.round(progress.elapsedMs / 1000)}s, ${progress.candidates ?? progress.items} candidates, ${progress.filesSized ?? 0} files sized, ${progress.directoriesSized ?? 0} directories sized (${progress.status})`
+      );
     return await dispatch(
       new Cli(io, deps.api ?? API),
       command,

package/src/docker/cli.js

@@ -306,12 +306,16 @@ export class DockerCli {
     const stream = this.executor.spawn(
       this.argv(['cp', `${id}:${source}`, '-'])
     );
-    return pipeInto(stream, [
-      'tar',
-      '-x',
-      '-C',
-      destination,
-      ...excludes.map((pattern) => `--exclude=${pattern}`),
-    ]);
+    return pipeInto(
+      stream,
+      [
+        'tar',
+        '-x',
+        '-C',
+        destination,
+        ...excludes.map((pattern) => `--exclude=${pattern}`),
+      ],
+      { signal: this.executor.signal }
+    );
   }
 }

package/src/docker/scan.js

@@ -22,6 +22,7 @@
 
 import { backupDirectory } from '../audit.js';
 import { ShellEnv } from '../env/shell.js';
+import { checkSignal, signalExecutor } from '../env/scope.js';
 import { containerExecutor, trace } from '../exec.js';
 import { block, makeItem } from '../items.js';
 import { DockerCli, parseDockerSize } from './cli.js';
@@ -380,6 +381,7 @@ class DockerScan {
   }
 
   async daemon(record, depth, inherited) {
+    checkSignal(this.options.signal);
     const docker = new DockerCli(record.executor);
     const info = await docker.info();
     if (!info) {
@@ -443,6 +445,7 @@ class DockerScan {
       })),
     };
     for (const ps of all) {
+      checkSignal(this.options.signal);
       ps.sizes = await this.bestEffort(
         record.env.id,
         () => docker.containerSize(ps.ID),
@@ -544,6 +547,7 @@ class DockerScan {
       }))
     );
     for (const ps of all) {
+      checkSignal(this.options.signal);
       const selected = selection.ids.has(ps.ID);
       if (!selected) {
         continue;
@@ -597,7 +601,10 @@ class DockerScan {
 
   async running(record, depth, ps, entry, inspected) {
     const label = `${record.env.label}/${entry.name}`;
-    const executor = containerExecutor(record.executor, ps.ID, { label });
+    const executor = signalExecutor(
+      containerExecutor(record.executor, ps.ID, { label }),
+      this.options.signal
+    );
     const env = new ShellEnv(executor, {
       id: `${record.env.id}/${entry.name}`,
       label,
@@ -614,14 +621,20 @@ class DockerScan {
     entry.env = env.id;
     entry.scanned = true;
     const chain = [...record.chain, { containerId: ps.ID, name: entry.name }];
-    this.result.environments.push({
+    const descriptor = {
       id: env.id,
       label,
       kind: 'container',
       depth,
       chain,
-    });
+    };
+    this.result.environments.push(descriptor);
     this.result.items.push(...(await this.scanEnvironment(env)));
+    Object.assign(descriptor, {
+      scanStatus: env.scanStatus,
+      hint: env.probeHint ?? null,
+      layerSource: env.writableLayer?.source ?? null,
+    });
     if (await env.which('docker')) {
       try {
         await this.daemon({ env, executor, chain }, depth, true);
@@ -650,6 +663,14 @@ export async function scanDocker(hostRecord, options, scanEnvironment) {
   if (!(await hostRecord.env.which('docker'))) {
     return scan.result;
   }
-  await scan.daemon(hostRecord, 0, false);
+  try {
+    await scan.daemon(hostRecord, 0, false);
+  } catch (error) {
+    scan.result.errors.push({
+      env: hostRecord.env.id,
+      scanner: 'docker',
+      message: error.message,
+    });
+  }
   return scan.result;
 }

package/src/docker/upperdir.js

@@ -0,0 +1,44 @@
+import { batches } from '../env/batches.js';
+
+/** Root overlay from the running container's mount namespace, read by its daemon. */
+export function rootUpperdir(mountinfo) {
+  for (const line of mountinfo.split('\n')) {
+    const [mount, filesystem] = line.split(' - ');
+    if (mount.split(' ')[4] !== '/' || !filesystem?.startsWith('overlay ')) {
+      continue;
+    }
+    const options = filesystem.split(' ').slice(2).join(' ');
+    const upper = /(?:^|,)upperdir=([^,]+)/.exec(options)?.[1];
+    if (upper) {
+      return upper.replace(/\\([0-7]{3})/g, (_, code) =>
+        String.fromCharCode(Number.parseInt(code, 8))
+      );
+    }
+  }
+  return null;
+}
+
+/** One transport call per bounded batch; null means inaccessible, never guessed. */
+export async function upperdirBytes(executor, paths) {
+  const measured = new Map();
+  const script = `for t do
+    printf '\\001%s\\n' "$t"
+    value=$(LC_ALL=C du -skx -- "$t" 2>&1)
+    code=$?
+    if [ "$code" = 0 ]; then printf '%s\\n' "$value" | cut -f1
+    else case "$value" in *'No such file or directory'*) echo 0 ;; *) echo unknown ;; esac; fi
+  done`;
+  for (const batch of batches(paths)) {
+    const result = await executor.run(['sh', '-c', script, 'dss', ...batch]);
+    for (const record of result.stdout.split('\u0001').slice(1)) {
+      const newline = record.indexOf('\n'),
+        target = record.slice(0, newline),
+        value = record.slice(newline + 1).trim();
+      measured.set(
+        target,
+        result.code === 0 && /^\d+$/.test(value) ? Number(value) * 1024 : null
+      );
+    }
+  }
+  return measured;
+}

package/src/docker/writable.js

@@ -3,11 +3,13 @@ import { DockerCli } from './cli.js';
 import { isWithin } from '../paths.js';
 import { trace } from '../exec.js';
 import { resetDiffSnapshots } from './snapshots.js';
+import { rootUpperdir, upperdirBytes } from './upperdir.js';
 
 let measurementQueue = Promise.resolve();
 const layers = new WeakMap();
 
 export function sharedWritableLayer(executor, id) {
+  executor = executor.unscoped ?? executor;
   if (!layers.has(executor)) {
     layers.set(executor, new Map());
   }
@@ -19,12 +21,14 @@ export function sharedWritableLayer(executor, id) {
 }
 
 export function resetWritableLayers(executor) {
+  executor = executor.unscoped ?? executor;
   layers.delete(executor);
   resetDiffSnapshots(executor);
 }
 
 /** Overlay without an authoritative upper layer must never count merged bytes. */
 export class UnknownWritableLayer {
+  source = 'unknown-overlay';
   measure(env, usages) {
     return Promise.resolve(
       new Map(
@@ -67,14 +71,14 @@ export class WritableLayer {
         if (!container) {
           return { diff: null, container };
         }
-        const upperdir =
-          container.GraphDriver?.Data?.UpperDir ?? this.upperdirCandidate;
+        const { upperdir, source } = await this.upperdir(container);
         if (upperdir?.startsWith('/')) {
           const result = await this.docker.executor
             .run(['stat', '-c', '%F', '--', upperdir])
             .catch(() => null);
           if (result?.code === 0 && result.stdout.trim() === 'directory') {
             trace('using authoritative upperdir', this.id);
+            this.source = source;
             return { container, upperdir, diff: null };
           }
         }
@@ -86,6 +90,7 @@ export class WritableLayer {
           return { diff: null, container };
         }
         const diff = await this.docker.diff(this.id);
+        this.source = diff === null ? 'unknown' : 'docker-diff';
         trace('finished measuring container', this.id);
         // Bound the retained change set; unknown is safer than a partial result.
         return {
@@ -102,6 +107,27 @@ export class WritableLayer {
     return this.snapshotPromise;
   }
 
+  async upperdir(container) {
+    if (container.GraphDriver?.Data?.UpperDir) {
+      return {
+        upperdir: container.GraphDriver.Data.UpperDir,
+        source: 'graphdriver-upperdir',
+      };
+    }
+    const pid = container.State?.Pid;
+    if (Number.isInteger(pid) && pid > 0) {
+      const info = await this.docker.executor.run([
+        'cat',
+        `/proc/${pid}/mountinfo`,
+      ]);
+      const upperdir = info.code === 0 ? rootUpperdir(info.stdout) : null;
+      if (upperdir) {
+        return { upperdir, source: 'mountinfo-upperdir' };
+      }
+    }
+    return { upperdir: this.upperdirCandidate, source: 'overlay-upperdir' };
+  }
+
   forget(target, pathApi) {
     this.removed.add(target);
     for (const file of this.stats.keys()) {
@@ -111,11 +137,55 @@ export class WritableLayer {
     }
   }
 
+  /** Allocated upper blocks before/after deletion, independent of merged data. */
+  async allocatedBytes(paths, pathApi) {
+    const { container, upperdir } = await this.snapshot();
+    if (
+      !upperdir ||
+      (container.Mounts ?? []).some((mount) =>
+        paths.some(
+          (target) =>
+            isWithin(target, mount.Destination, pathApi) ||
+            isWithin(mount.Destination, target, pathApi)
+        )
+      )
+    ) {
+      return null;
+    }
+    const measured = await upperdirBytes(
+      this.docker.executor,
+      paths.map((target) => `${upperdir}/${target.replace(/^\/+/, '')}`)
+    );
+    const values = [...measured.values()];
+    return values.length === paths.length &&
+      values.every((value) => value !== null)
+      ? values.reduce((sum, value) => sum + value, 0)
+      : null;
+  }
+
   async measureUpperdir(env, usages, container, upperdir) {
     const result = new Map();
     const mounts = (container.Mounts ?? [])
       .map((mount) => mount.Destination)
       .filter(Boolean);
+    const upperPath = (target) => `${upperdir}/${target.replace(/^\/+/, '')}`;
+    const targets = [...usages.keys()].filter(
+      (target) => !mounts.some((mount) => isWithin(target, mount, env.path))
+    );
+    const measurements = await upperdirBytes(
+      this.docker.executor,
+      targets.map(upperPath)
+    );
+    const mountRoots = mounts.filter(
+      (mount) =>
+        targets.some((target) => isWithin(mount, target, env.path)) &&
+        !mounts.some(
+          (other) => other !== mount && isWithin(mount, other, env.path)
+        )
+    );
+    const mountedUsage = mountRoots.length
+      ? await env.rawUsageMany(mountRoots)
+      : new Map();
     for (const [target, usage] of usages) {
       if (mounts.some((mount) => isWithin(target, mount, env.path))) {
         result.set(target, {
@@ -125,17 +195,24 @@ export class WritableLayer {
         });
         continue;
       }
-      const measured = await this.docker.executor.run([
-        'du',
-        '-skx',
-        '--',
-        `${upperdir}/${target.replace(/^\/+/, '')}`,
-      ]);
-      const kib = Number(/^([0-9]+)\s/m.exec(measured.stdout)?.[1]);
-      // Missing upperdir leaf means image-only. Other errors remain unknown.
-      const absent = /no such file|not found/i.test(measured.stderr);
-      const known = (measured.code === 0 && Number.isFinite(kib)) || absent;
-      const bytes = known ? Math.min(usage.bytes, absent ? 0 : kib * 1024) : 0;
+      const measured = measurements.get(upperPath(target));
+      const contained = mountRoots.filter((mount) =>
+        isWithin(mount, target, env.path)
+      );
+      const known =
+        measured !== null &&
+        measured !== undefined &&
+        contained.every((mount) => mountedUsage.has(mount));
+      const bytes = known
+        ? Math.min(
+            usage.bytes,
+            measured +
+              contained.reduce(
+                (sum, mount) => sum + mountedUsage.get(mount).bytes,
+                0
+              )
+          )
+        : 0;
       result.set(target, {
         ...usage,
         bytes,

package/src/emergency.js

@@ -89,12 +89,40 @@ export async function emergency(input = {}) {
   });
   const report = input.report
     ? await revalidateReport(input.report, { ...input, env })
-    : await scan({ ...input, env });
-  audit.report = { createdAt: report.createdAt, totals: report.totals };
+    : goalMet(goal, diskBefore)
+      ? {
+          createdAt: new Date(options.now()).toISOString(),
+          items: [],
+          environments: [{ id: env.id, label: env.label }],
+          totals: {},
+        }
+      : await scan({
+          ...input,
+          env,
+          onScanPhase: async (phase) => {
+            audit.diskAfter = await escalate({
+              report: phase,
+              options: {
+                ...options,
+                tier: 'safe',
+                streaming: options.docker !== false,
+              },
+              env,
+              target,
+              goal,
+              audit,
+            });
+            return !goalMet(goal, audit.diskAfter);
+          },
+        });
+  audit.report = report;
   if (options.audit !== false) {
     await writeAudit(audit, { ...options, inProgress: true });
   }
-  let disk = diskBefore;
+  let disk = audit.diskAfter ?? diskBefore;
+  if (report.aborted) {
+    audit.scanReport = report;
+  }
   try {
     if (!goalMet(goal, disk)) {
       disk = await escalate({ report, options, env, target, goal, audit });
@@ -121,9 +149,10 @@ async function escalate({ report, options, env, target, goal, audit }) {
     ...options,
     env,
     onProgress: persistProgress,
+    priorHealth: audit.health,
   });
   audit.health = cleaner.health.records;
-  const done = new Set();
+  const done = new Set(audit.entries.map((entry) => entry.id));
   const targetDevice = await env.deviceId?.(target);
   const scoped = await Promise.all(
     wantedItems(report, options).map(async (item) => {
@@ -141,8 +170,10 @@ async function escalate({ report, options, env, target, goal, audit }) {
     })
   );
   const wanted = scoped.filter(Boolean);
-  let disk = audit.diskBefore;
-  let freed = 0;
+  let disk = audit.diskAfter ?? audit.diskBefore;
+  let freed = audit.entries
+    .filter((entry) => entry.status === 'planned')
+    .reduce((sum, entry) => sum + entry.plannedBytes, 0);
   for (const tier of TIERS.slice(0, tierRank(options.tier) + 1)) {
     audit.reachedTier = tier;
     const items = selectByTier(wanted, tier)

package/src/env/batches.js

@@ -0,0 +1,20 @@
+/** Bound argument count and encoded size below common exec limits. */
+export function batches(items, size = 200, maxBytes = 48000) {
+  const result = [];
+  let batch = [],
+    bytes = 0;
+  for (const item of items) {
+    const length = Buffer.byteLength(String(item)) + 1;
+    if (batch.length && (batch.length >= size || bytes + length > maxBytes)) {
+      result.push(batch);
+      batch = [];
+      bytes = 0;
+    }
+    batch.push(item);
+    bytes += length;
+  }
+  if (batch.length) {
+    result.push(batch);
+  }
+  return result;
+}