@git.zone/tsdisk 1.2.1 → 1.4.0

package/ts/tsdisk.usage.ts

@@ -0,0 +1,199 @@
+import * as plugins from './tsdisk.plugins.js';
+
+export interface IUsageEntry {
+  path: string;
+  bytes: number;
+  complete: boolean;
+}
+
+export interface IUsageReport {
+  roots: string[];
+  measuredBytes: number;
+  entries: IUsageEntry[];
+  records: number;
+  complete: boolean;
+  code: number;
+  stderr: string;
+}
+
+export function containsPath(parent: string, child: string): boolean {
+  const relative = plugins.path.relative(parent, child);
+  return relative === '' || (!relative.startsWith(`..${plugins.path.sep}`) && relative !== '..' && !plugins.path.isAbsolute(relative));
+}
+
+/** Preserve explicitly selected nested mounts; subsume overlapping roots on the same device. */
+export async function normalizeScanRoots(roots: string[]): Promise<string[]> {
+  const unique = [...new Set(roots.map((root) => plugins.path.resolve(root)))].sort((a, b) => a.length - b.length || a.localeCompare(b));
+  const accepted: Array<{ path: string; dev?: number }> = [];
+  for (const path of unique) {
+    const stat = await plugins.fs.lstat(path).catch(() => undefined);
+    const real = await plugins.fs.realpath(path).catch(() => undefined);
+    const dev = stat?.isDirectory() && real === path ? stat.dev : undefined;
+    if (dev !== undefined && accepted.some((parent) => parent.dev === dev && containsPath(parent.path, path))) continue;
+    accepted.push({ path, dev });
+  }
+  return accepted.map((root) => root.path);
+}
+
+/** GNU du emits files immediately and inclusive directory totals on completion.
+ * Reconcile each completed directory with its observed children, rather than
+ * adding inclusive totals twice. The active ancestor chain survives a timeout.
+ */
+export class UsageAccumulator {
+  private pending = '';
+  private active = new Map<string, number>();
+  private largest = new Map<string, IUsageEntry>();
+  private tracked = new Map<string, IUsageEntry>();
+  private minimumLargestBytes = 0;
+  private operandBytes = new Map<string, number>();
+  private accountingRoots: string[];
+  private currentRoot?: string;
+  public measuredBytes = 0;
+  public records = 0;
+  public currentPath = '';
+
+  constructor(private roots: string[], private trackedPaths: Set<string> = new Set(), private priorityPaths: string[] = []) {
+    this.roots = [...roots].sort((a, b) => b.length - a.length);
+    if (roots.some((root) => plugins.path.resolve(root) !== root) || priorityPaths.some((path) => !roots.some((root) => root !== path && containsPath(root, path)))) {
+      throw new Error('Usage accounting requires normalized roots and strict descendant priority paths.');
+    }
+    this.accountingRoots = [...priorityPaths, ...roots].sort((a, b) => b.length - a.length);
+    this.trackedPaths = new Set([...trackedPaths, ...roots, ...priorityPaths]);
+  }
+
+  public consume(chunk: string) {
+    this.pending += chunk;
+    let start = 0;
+    for (;;) {
+      const end = this.pending.indexOf('\0', start);
+      if (end < 0) break;
+      this.record(this.pending.slice(start, end));
+      start = end + 1;
+    }
+    this.pending = this.pending.slice(start);
+    if (this.pending.length > 1024 * 1024) throw new Error('Invalid du record exceeds 1 MiB.');
+  }
+
+  private record(record: string) {
+    const separator = record.indexOf('\t');
+    const bytes = Number(record.slice(0, separator));
+    const path = record.slice(separator + 1);
+    const matches = (candidate: string) => path === candidate || path.startsWith(candidate === '/' ? '/' : `${candidate}/`);
+    const root = this.currentRoot && matches(this.currentRoot) ? this.currentRoot : this.accountingRoots.find(matches);
+    if (separator < 1 || !Number.isSafeInteger(bytes) || bytes < 0 || !plugins.path.isAbsolute(path) || !root) throw new Error('Invalid du output.');
+    this.records++;
+    this.currentPath = path;
+    this.currentRoot = path === root ? undefined : root;
+    const delta = bytes - (this.active.get(path) ?? 0);
+    this.active.delete(path);
+    this.measuredBytes += delta;
+    this.operandBytes.set(root, (this.operandBytes.get(root) ?? 0) + delta);
+    if (path !== root) {
+      for (let parent = plugins.path.dirname(path); ; parent = plugins.path.dirname(parent)) {
+        this.active.set(parent, (this.active.get(parent) ?? 0) + delta);
+        if (parent === root) break;
+      }
+    }
+    const entry = { path, bytes, complete: true };
+    if (this.trackedPaths.has(path)) this.tracked.set(path, entry);
+    // Retain only the largest completed paths, not millions of file records.
+    if (this.largest.size < 100 || bytes > this.minimumLargestBytes) {
+      this.largest.set(path, entry);
+      if (this.largest.size > 100) {
+        const sorted = [...this.largest.values()].sort((a, b) => b.bytes - a.bytes);
+        this.largest = new Map(sorted.slice(0, 100).map((item) => [item.path, item]));
+      }
+      this.minimumLargestBytes = Math.min(...[...this.largest.values()].map((item) => item.bytes));
+    }
+  }
+
+  public snapshot(): IUsageEntry[] {
+    const entries = new Map([...this.largest, ...this.tracked].map(([path, entry]) => [path, { ...entry }]));
+    for (const [path, bytes] of this.active) entries.set(path, { path, bytes, complete: false });
+    // du charges an overlapping subtree to its first operand and omits it from
+    // later ancestor totals. Restore those inclusive totals for display only;
+    // measuredBytes remains the native, deduplicated sum of operand contributions.
+    for (const priority of this.priorityPaths) {
+      const bytes = this.operandBytes.get(priority);
+      const root = this.roots.find((candidate) => containsPath(candidate, priority));
+      if (bytes === undefined || !root) continue;
+      for (let parent = plugins.path.dirname(priority); ; parent = plugins.path.dirname(parent)) {
+        const entry = entries.get(parent) ?? { path: parent, bytes: 0, complete: false };
+        entry.bytes += bytes;
+        entries.set(parent, entry);
+        if (parent === root) break;
+      }
+    }
+    return [...entries.values()].sort((a, b) => b.bytes - a.bytes);
+  }
+
+  public get endedOnRecordBoundary() { return this.pending.length === 0; }
+}
+
+export interface IUsageOptions {
+  timeoutSeconds: number;
+  trackedPaths?: string[];
+  priorityPaths?: string[];
+  onProgress?: (usage: UsageAccumulator) => void;
+}
+
+/** One streaming native pass; no output export, no symlink traversal, no raw file content reads. */
+export async function measureDiskUsage(roots: string[], options: IUsageOptions): Promise<IUsageReport> {
+  if (!Number.isFinite(options.timeoutSeconds) || options.timeoutSeconds <= 0) throw new Error('Invalid usage timeout.');
+  const valid: string[] = [];
+  const devices = new Map<string, number>();
+  const errors: string[] = [];
+  for (const root of await normalizeScanRoots(roots)) {
+    try {
+      const stat = await plugins.fs.lstat(root);
+      if (!stat.isDirectory() || await plugins.fs.realpath(root) !== root) throw new Error('Usage roots must be real directories without symlink ancestors.');
+      valid.push(root);
+      devices.set(root, stat.dev);
+    } catch (error) { errors.push(`${root}: ${error instanceof Error ? error.message : String(error)}`); }
+  }
+  const priorities: string[] = [];
+  for (const path of await normalizeScanRoots(options.priorityPaths ?? [])) {
+    const root = [...valid].reverse().find((root) => root !== path && containsPath(root, path));
+    if (!root) continue;
+    const real = await plugins.fs.realpath(path).catch(() => undefined);
+    const stat = await plugins.fs.lstat(path).catch(() => undefined);
+    if (real === path && stat?.isDirectory() && stat.dev === devices.get(root)) priorities.push(path);
+  }
+  // Measure retained project artifacts before broad dependency/source trees.
+  // This changes order only: remaining paths are visited by the final roots.
+  priorities.sort((a, b) => b.split(plugins.path.sep).length - a.split(plugins.path.sep).length || a.localeCompare(b));
+  let argumentBytes = valid.reduce((sum, path) => sum + Buffer.byteLength(path) + 1, 0);
+  const argumentLimit = priorities.findIndex((path) => (argumentBytes += Buffer.byteLength(path) + 1) > 128 * 1024);
+  if (argumentLimit >= 0) priorities.splice(argumentLimit);
+  const usage = new UsageAccumulator(valid, new Set(options.trackedPaths), priorities);
+  let stderr = errors.join('\n');
+  let malformed = false;
+  const code = valid.length ? await new Promise<number>((resolve) => {
+    const child = plugins.spawn('timeout', ['--kill-after=5s', `${options.timeoutSeconds}s`, 'du', '-a', '-0', '-x', '-B1', '--', ...priorities, ...valid], { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, LC_ALL: 'C' } });
+    child.stdout.setEncoding('utf8');
+    let lastNotification = 0;
+    child.stdout.on('data', (chunk: string) => {
+      if (malformed) return;
+      try {
+        usage.consume(chunk);
+        if (Date.now() - lastNotification >= 250) {
+          options.onProgress?.(usage);
+          lastNotification = Date.now();
+        }
+      } catch (error) {
+        malformed = true;
+        stderr += `\n${error instanceof Error ? error.message : String(error)}`;
+        // Signal timeout, which forwards termination to its managed process group.
+        child.kill('SIGTERM');
+      }
+    });
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (chunk: string) => { if (stderr.length < 65536) stderr += chunk.slice(0, 65536 - stderr.length); });
+    child.on('error', (error) => { stderr += `\n${error.message}`; });
+    child.on('close', (exitCode) => resolve(exitCode ?? 1));
+  }) : 1;
+  options.onProgress?.(usage);
+  const complete = code === 0 && !stderr.trim() && !malformed && usage.endedOnRecordBoundary;
+  const entries = usage.snapshot().map((entry) => ({ ...entry, complete: entry.complete && complete }));
+  return { roots: valid, measuredBytes: usage.measuredBytes, entries, records: usage.records, complete, code, stderr };
+}