@git.zone/tsdisk 1.1.0 → 1.3.0

package/ts/tsdisk.scan.ts

@@ -0,0 +1,151 @@
+import * as plugins from './tsdisk.plugins.js';
+import { classifyToolCachePath, parseToolCacheMarker, toolCacheMarkerFile, type IToolCacheFinding } from './tsdisk.toolcaches.js';
+
+export interface IWorkspaceScanOptions {
+  maxDepth: number;
+  timeoutMs: number;
+  maxEntries: number;
+  onProgress?: (result: IWorkspaceScanResult) => void;
+}
+
+export interface IScanIssue {
+  path: string;
+  reason: 'read-error' | 'invalid-marker' | 'depth-limit' | 'entry-limit' | 'timeout' | 'mount-boundary' | 'symlink' | 'size-unavailable';
+  detail: string;
+}
+
+export interface IWorkspaceScanResult {
+  root: string;
+  findings: IToolCacheFinding[];
+  issues: IScanIssue[];
+  complete: boolean;
+  visitedEntries: number;
+  issueCounts: Partial<Record<IScanIssue['reason'], number>>;
+  usagePriorityPaths: string[];
+}
+
+const excludedDirectories = new Set(['.git', 'node_modules', 'dist_ts', 'dist_ts_web', 'coverage', '.pnpm-store']);
+
+export async function scanWorkspace(root: string, options: IWorkspaceScanOptions): Promise<IWorkspaceScanResult> {
+  for (const name of ['maxDepth', 'timeoutMs', 'maxEntries'] as const) {
+    const value = options[name];
+    if (!Number.isSafeInteger(value) || value < (name === 'maxDepth' ? 0 : 1)) {
+      throw new Error(`Invalid scan option ${name}: ${value}`);
+    }
+  }
+  const result: IWorkspaceScanResult = { root: plugins.path.resolve(root), findings: [], issues: [], issueCounts: {}, usagePriorityPaths: [], complete: true, visitedEntries: 0 };
+  const deadline = Date.now() + options.timeoutMs;
+  let stopped = false;
+  const issue = (path: string, reason: IScanIssue['reason'], detail: string) => {
+    result.complete = false;
+    result.issueCounts[reason] = (result.issueCounts[reason] ?? 0) + 1;
+    if (result.issues.length < 200) result.issues.push({ path, reason, detail });
+  };
+  const limited = (path: string) => {
+    if (stopped) return true;
+    if (Date.now() >= deadline || result.visitedEntries >= options.maxEntries) {
+      issue(path, Date.now() >= deadline ? 'timeout' : 'entry-limit', 'Scan stopped before visiting all entries.');
+      stopped = true;
+    }
+    return stopped;
+  };
+  let rootDev: number;
+  try {
+    const stat = await plugins.fs.lstat(result.root);
+    if (stat.isSymbolicLink() || await plugins.fs.realpath(result.root) !== result.root) {
+      issue(result.root, 'symlink', 'Scan roots must not contain symbolic links.');
+      return result;
+    }
+    if (!stat.isDirectory()) throw new Error('Scan root is not a directory.');
+    rootDev = stat.dev;
+  } catch (error) {
+    issue(result.root, 'read-error', error instanceof Error ? error.message : String(error));
+    return result;
+  }
+  const queue: Array<{ directory: string; depth: number }> = [{ directory: result.root, depth: 0 }];
+  let queueLimited = false;
+  const visit = async (directory: string, depth: number): Promise<void> => {
+    if (limited(directory)) return;
+    result.visitedEntries++;
+    try {
+      const stat = await plugins.fs.lstat(directory);
+      if (stat.isSymbolicLink()) {
+        issue(directory, 'symlink', 'Symbolic link skipped.');
+        return;
+      }
+      if (stat.dev !== rootDev) {
+        issue(directory, 'mount-boundary', 'Different filesystem skipped.');
+        return;
+      }
+      if (plugins.path.basename(directory) === '.nogit') result.usagePriorityPaths.push(directory);
+      let marker;
+      const markerPath = plugins.path.join(directory, toolCacheMarkerFile);
+      const markerStat = await plugins.fs.lstat(markerPath).catch((error: NodeJS.ErrnoException) => {
+        if (error.code === 'ENOENT') return undefined;
+        throw error;
+      });
+      if (markerStat) {
+        if (!markerStat.isFile() || markerStat.size > 65536) {
+          issue(markerPath, 'invalid-marker', 'Marker must be a regular file of at most 64 KiB.');
+        } else {
+          marker = parseToolCacheMarker(await plugins.fs.readFile(markerPath, 'utf8'));
+          if (!marker) issue(markerPath, 'invalid-marker', 'Invalid cache ownership marker.');
+        }
+      }
+      const heuristic = classifyToolCachePath(directory);
+      const classification = classifyToolCachePath(directory, marker);
+      if (classification) result.findings.push({ path: directory, ...classification });
+      // Cache contents are not workspaces. Registry session directories can carry their own markers.
+      if (classification && heuristic?.kind !== 'docker-registry-cache') return;
+      let enumerated = 0;
+      const children = await plugins.fs.opendir(directory);
+      for await (const child of children) {
+        // A huge flat directory must not consume the budget of sibling projects.
+        if (++enumerated > 4096) {
+          issue(directory, 'entry-limit', 'Directory listing stopped at 4096 entries; sibling directories are still scanned.');
+          break;
+        }
+        if (excludedDirectories.has(child.name)) continue;
+        if (!child.isDirectory() && !child.isSymbolicLink()) continue;
+        if (depth >= options.maxDepth) {
+          issue(directory, 'depth-limit', `Maximum scan depth ${options.maxDepth} reached.`);
+          break;
+        }
+        const childPath = plugins.path.join(directory, child.name);
+        if (limited(childPath)) return;
+        if (child.isSymbolicLink()) {
+          result.visitedEntries++;
+          issue(childPath, 'symlink', 'Symbolic link skipped.');
+          continue;
+        }
+        if (heuristic?.kind === 'docker-registry-cache') {
+          // Only direct sessions, never registry blob trees.
+          const sessionMarker = await plugins.fs.lstat(plugins.path.join(childPath, toolCacheMarkerFile)).catch((error: NodeJS.ErrnoException) => {
+            if (error.code === 'ENOENT') return undefined;
+            throw error;
+          });
+          if (!sessionMarker) { result.visitedEntries++; continue; }
+        }
+        if (queue.length >= options.maxEntries) {
+          if (!queueLimited) issue(directory, 'entry-limit', 'Directory queue budget exhausted; already queued siblings are still scanned.');
+          queueLimited = true;
+          continue;
+        }
+        queue.push({ directory: childPath, depth: depth + 1 });
+      }
+    } catch (error) {
+      issue(directory, 'read-error', error instanceof Error ? error.message : String(error));
+    }
+  };
+  // Breadth first: canonical projects precede deeply nested retained worktrees.
+  let lastProgress = 0;
+  for (let index = 0; index < queue.length && !stopped; index++) {
+    await visit(queue[index].directory, queue[index].depth);
+    if (Date.now() - lastProgress >= 250) {
+      options.onProgress?.(result);
+      lastProgress = Date.now();
+    }
+  }
+  options.onProgress?.(result);
+  return result;
+}

package/ts/tsdisk.toolcaches.ts

@@ -1,4 +1,4 @@
-import * as path from 'node:path';
+import { path } from './tsdisk.plugins.js';
 
 export const toolCacheMarkerFile = '.gitzone-tool-cache.json';
 
@@ -29,10 +29,10 @@ export function parseToolCacheMarker(rawArg: string): IToolCacheMarker | undefin
   try {
     const marker = JSON.parse(rawArg) as Partial<IToolCacheMarker>;
     if (
-      typeof marker.owner === 'string' &&
-      typeof marker.kind === 'string' &&
-      marker.safeToPrune === true &&
-      typeof marker.createdAt === 'string' &&
+      typeof marker.owner === 'string' && marker.owner.length > 0 &&
+      typeof marker.kind === 'string' && marker.kind.length > 0 &&
+      typeof marker.safeToPrune === 'boolean' &&
+      typeof marker.createdAt === 'string' && Number.isFinite(Date.parse(marker.createdAt)) &&
       marker.schemaVersion === 1
     ) {
       return marker as IToolCacheMarker;
@@ -79,8 +79,10 @@ export function classifyToolCachePath(pathArg: string, markerArg?: IToolCacheMar
     return {
       ownerGuess: markerArg.owner,
       kind: markerArg.kind,
-      risk: markerArg.safeToPrune ? 'safe-marked' : 'review',
-      suggestedCleanupCommand: `rm -rf ${quotePath(pathArg)}`,
+      risk: 'review',
+      suggestedCleanupCommand: markerArg.safeToPrune
+        ? `report-only: ownership marker is not proof of inactivity; use the owning tool to review ${quotePath(pathArg)}`
+        : `protected: owner explicitly disallows pruning ${quotePath(pathArg)}`,
       marker: markerArg,
     };
   }
@@ -103,7 +105,7 @@ export function classifyToolCachePath(pathArg: string, markerArg?: IToolCacheMar
     };
   }
 
-  if (normalized.endsWith('/.nogit/tsbundle-temp')) {
+  if (/\/\.nogit\/tsbundle-temp(?:-[^/]+)?$/.test(normalized)) {
     return {
       ownerGuess: '@git.zone/tsbundle',
       kind: 'bundle-temp-workspace',
@@ -112,6 +114,24 @@ export function classifyToolCachePath(pathArg: string, markerArg?: IToolCacheMar
     };
   }
 
+  if (/\/\.nogit\/tsrust-matrix(?:-archive|-probes)?$/.test(normalized)) {
+    return {
+      ownerGuess: '@git.zone/tsrust',
+      kind: 'rust-matrix-artifacts',
+      risk: 'report-only',
+      suggestedCleanupCommand: `report-only: retained matrix runs may be needed for recovery; inspect ${quotePath(pathArg)}`,
+    };
+  }
+
+  if (/\/\.nogit\/(?:docker-registry\.[^/]+|docker-registry-v1-cache|dockerimagestore)$/.test(normalized)) {
+    return {
+      ownerGuess: '@git.zone/tsdocker',
+      kind: 'legacy-docker-registry-cache',
+      risk: 'report-only',
+      suggestedCleanupCommand: `report-only: verify release recovery requirements before removing ${quotePath(pathArg)}`,
+    };
+  }
+
   if (normalized.endsWith('/.nogit/docker-registry') || normalized.includes('/.nogit/docker-registry/')) {
     return {
       ownerGuess: '@git.zone/tsdocker',

package/ts/tsdisk.usage.ts

@@ -0,0 +1,199 @@
+import * as plugins from './tsdisk.plugins.js';
+
+export interface IUsageEntry {
+  path: string;
+  bytes: number;
+  complete: boolean;
+}
+
+export interface IUsageReport {
+  roots: string[];
+  measuredBytes: number;
+  entries: IUsageEntry[];
+  records: number;
+  complete: boolean;
+  code: number;
+  stderr: string;
+}
+
+export function containsPath(parent: string, child: string): boolean {
+  const relative = plugins.path.relative(parent, child);
+  return relative === '' || (!relative.startsWith(`..${plugins.path.sep}`) && relative !== '..' && !plugins.path.isAbsolute(relative));
+}
+
+/** Preserve explicitly selected nested mounts; subsume overlapping roots on the same device. */
+export async function normalizeScanRoots(roots: string[]): Promise<string[]> {
+  const unique = [...new Set(roots.map((root) => plugins.path.resolve(root)))].sort((a, b) => a.length - b.length || a.localeCompare(b));
+  const accepted: Array<{ path: string; dev?: number }> = [];
+  for (const path of unique) {
+    const stat = await plugins.fs.lstat(path).catch(() => undefined);
+    const real = await plugins.fs.realpath(path).catch(() => undefined);
+    const dev = stat?.isDirectory() && real === path ? stat.dev : undefined;
+    if (dev !== undefined && accepted.some((parent) => parent.dev === dev && containsPath(parent.path, path))) continue;
+    accepted.push({ path, dev });
+  }
+  return accepted.map((root) => root.path);
+}
+
+/** GNU du emits files immediately and inclusive directory totals on completion.
+ * Reconcile each completed directory with its observed children, rather than
+ * adding inclusive totals twice. The active ancestor chain survives a timeout.
+ */
+export class UsageAccumulator {
+  private pending = '';
+  private active = new Map<string, number>();
+  private largest = new Map<string, IUsageEntry>();
+  private tracked = new Map<string, IUsageEntry>();
+  private minimumLargestBytes = 0;
+  private operandBytes = new Map<string, number>();
+  private accountingRoots: string[];
+  private currentRoot?: string;
+  public measuredBytes = 0;
+  public records = 0;
+  public currentPath = '';
+
+  constructor(private roots: string[], private trackedPaths: Set<string> = new Set(), private priorityPaths: string[] = []) {
+    this.roots = [...roots].sort((a, b) => b.length - a.length);
+    if (roots.some((root) => plugins.path.resolve(root) !== root) || priorityPaths.some((path) => !roots.some((root) => root !== path && containsPath(root, path)))) {
+      throw new Error('Usage accounting requires normalized roots and strict descendant priority paths.');
+    }
+    this.accountingRoots = [...priorityPaths, ...roots].sort((a, b) => b.length - a.length);
+    this.trackedPaths = new Set([...trackedPaths, ...roots, ...priorityPaths]);
+  }
+
+  public consume(chunk: string) {
+    this.pending += chunk;
+    let start = 0;
+    for (;;) {
+      const end = this.pending.indexOf('\0', start);
+      if (end < 0) break;
+      this.record(this.pending.slice(start, end));
+      start = end + 1;
+    }
+    this.pending = this.pending.slice(start);
+    if (this.pending.length > 1024 * 1024) throw new Error('Invalid du record exceeds 1 MiB.');
+  }
+
+  private record(record: string) {
+    const separator = record.indexOf('\t');
+    const bytes = Number(record.slice(0, separator));
+    const path = record.slice(separator + 1);
+    const matches = (candidate: string) => path === candidate || path.startsWith(candidate === '/' ? '/' : `${candidate}/`);
+    const root = this.currentRoot && matches(this.currentRoot) ? this.currentRoot : this.accountingRoots.find(matches);
+    if (separator < 1 || !Number.isSafeInteger(bytes) || bytes < 0 || !plugins.path.isAbsolute(path) || !root) throw new Error('Invalid du output.');
+    this.records++;
+    this.currentPath = path;
+    this.currentRoot = path === root ? undefined : root;
+    const delta = bytes - (this.active.get(path) ?? 0);
+    this.active.delete(path);
+    this.measuredBytes += delta;
+    this.operandBytes.set(root, (this.operandBytes.get(root) ?? 0) + delta);
+    if (path !== root) {
+      for (let parent = plugins.path.dirname(path); ; parent = plugins.path.dirname(parent)) {
+        this.active.set(parent, (this.active.get(parent) ?? 0) + delta);
+        if (parent === root) break;
+      }
+    }
+    const entry = { path, bytes, complete: true };
+    if (this.trackedPaths.has(path)) this.tracked.set(path, entry);
+    // Retain only the largest completed paths, not millions of file records.
+    if (this.largest.size < 100 || bytes > this.minimumLargestBytes) {
+      this.largest.set(path, entry);
+      if (this.largest.size > 100) {
+        const sorted = [...this.largest.values()].sort((a, b) => b.bytes - a.bytes);
+        this.largest = new Map(sorted.slice(0, 100).map((item) => [item.path, item]));
+      }
+      this.minimumLargestBytes = Math.min(...[...this.largest.values()].map((item) => item.bytes));
+    }
+  }
+
+  public snapshot(): IUsageEntry[] {
+    const entries = new Map([...this.largest, ...this.tracked].map(([path, entry]) => [path, { ...entry }]));
+    for (const [path, bytes] of this.active) entries.set(path, { path, bytes, complete: false });
+    // du charges an overlapping subtree to its first operand and omits it from
+    // later ancestor totals. Restore those inclusive totals for display only;
+    // measuredBytes remains the native, deduplicated sum of operand contributions.
+    for (const priority of this.priorityPaths) {
+      const bytes = this.operandBytes.get(priority);
+      const root = this.roots.find((candidate) => containsPath(candidate, priority));
+      if (bytes === undefined || !root) continue;
+      for (let parent = plugins.path.dirname(priority); ; parent = plugins.path.dirname(parent)) {
+        const entry = entries.get(parent) ?? { path: parent, bytes: 0, complete: false };
+        entry.bytes += bytes;
+        entries.set(parent, entry);
+        if (parent === root) break;
+      }
+    }
+    return [...entries.values()].sort((a, b) => b.bytes - a.bytes);
+  }
+
+  public get endedOnRecordBoundary() { return this.pending.length === 0; }
+}
+
+export interface IUsageOptions {
+  timeoutSeconds: number;
+  trackedPaths?: string[];
+  priorityPaths?: string[];
+  onProgress?: (usage: UsageAccumulator) => void;
+}
+
+/** One streaming native pass; no output export, no symlink traversal, no raw file content reads. */
+export async function measureDiskUsage(roots: string[], options: IUsageOptions): Promise<IUsageReport> {
+  if (!Number.isFinite(options.timeoutSeconds) || options.timeoutSeconds <= 0) throw new Error('Invalid usage timeout.');
+  const valid: string[] = [];
+  const devices = new Map<string, number>();
+  const errors: string[] = [];
+  for (const root of await normalizeScanRoots(roots)) {
+    try {
+      const stat = await plugins.fs.lstat(root);
+      if (!stat.isDirectory() || await plugins.fs.realpath(root) !== root) throw new Error('Usage roots must be real directories without symlink ancestors.');
+      valid.push(root);
+      devices.set(root, stat.dev);
+    } catch (error) { errors.push(`${root}: ${error instanceof Error ? error.message : String(error)}`); }
+  }
+  const priorities: string[] = [];
+  for (const path of await normalizeScanRoots(options.priorityPaths ?? [])) {
+    const root = [...valid].reverse().find((root) => root !== path && containsPath(root, path));
+    if (!root) continue;
+    const real = await plugins.fs.realpath(path).catch(() => undefined);
+    const stat = await plugins.fs.lstat(path).catch(() => undefined);
+    if (real === path && stat?.isDirectory() && stat.dev === devices.get(root)) priorities.push(path);
+  }
+  // Measure retained project artifacts before broad dependency/source trees.
+  // This changes order only: remaining paths are visited by the final roots.
+  priorities.sort((a, b) => b.split(plugins.path.sep).length - a.split(plugins.path.sep).length || a.localeCompare(b));
+  let argumentBytes = valid.reduce((sum, path) => sum + Buffer.byteLength(path) + 1, 0);
+  const argumentLimit = priorities.findIndex((path) => (argumentBytes += Buffer.byteLength(path) + 1) > 128 * 1024);
+  if (argumentLimit >= 0) priorities.splice(argumentLimit);
+  const usage = new UsageAccumulator(valid, new Set(options.trackedPaths), priorities);
+  let stderr = errors.join('\n');
+  let malformed = false;
+  const code = valid.length ? await new Promise<number>((resolve) => {
+    const child = plugins.spawn('timeout', ['--kill-after=5s', `${options.timeoutSeconds}s`, 'du', '-a', '-0', '-x', '-B1', '--', ...priorities, ...valid], { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, LC_ALL: 'C' } });
+    child.stdout.setEncoding('utf8');
+    let lastNotification = 0;
+    child.stdout.on('data', (chunk: string) => {
+      if (malformed) return;
+      try {
+        usage.consume(chunk);
+        if (Date.now() - lastNotification >= 250) {
+          options.onProgress?.(usage);
+          lastNotification = Date.now();
+        }
+      } catch (error) {
+        malformed = true;
+        stderr += `\n${error instanceof Error ? error.message : String(error)}`;
+        // Signal timeout, which forwards termination to its managed process group.
+        child.kill('SIGTERM');
+      }
+    });
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', (chunk: string) => { if (stderr.length < 65536) stderr += chunk.slice(0, 65536 - stderr.length); });
+    child.on('error', (error) => { stderr += `\n${error.message}`; });
+    child.on('close', (exitCode) => resolve(exitCode ?? 1));
+  }) : 1;
+  options.onProgress?.(usage);
+  const complete = code === 0 && !stderr.trim() && !malformed && usage.endedOnRecordBoundary;
+  const entries = usage.snapshot().map((entry) => ({ ...entry, complete: entry.complete && complete }));
+  return { roots: valid, measuredBytes: usage.measuredBytes, entries, records: usage.records, complete, code, stderr };
+}