d4c-pool 0.2.0 → 0.3.0

package/dist/index.js

@@ -4,7 +4,7 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../core/src/reap/index.ts
 import { lstat as lstat2, stat as stat2, rm } from "node:fs/promises";
-import { basename, join as join3 } from "node:path";
+import { basename, join as join4 } from "node:path";
 
 // ../core/src/fs/trees.ts
 import { readdir, lstat, stat, readFile } from "node:fs/promises";
@@ -352,6 +352,80 @@ function restoreCommand(lockfile) {
   }
 }
 
+// ../core/src/fs/native.ts
+import { execFile } from "node:child_process";
+import { existsSync } from "node:fs";
+import { join as join3, dirname as dirname3 } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createRequire as createRequire2 } from "node:module";
+function findBinary(explicit) {
+  if (explicit !== undefined)
+    return existsSync(explicit) ? explicit : null;
+  if (process.env.D4C_FS_BIN !== undefined) {
+    return existsSync(process.env.D4C_FS_BIN) ? process.env.D4C_FS_BIN : null;
+  }
+  const pkg = `@voidmatcha/d4c-fs-${process.platform}-${process.arch}`;
+  try {
+    const req = createRequire2(import.meta.url);
+    return req.resolve(`${pkg}/d4c-fs`);
+  } catch {}
+  const here = dirname3(fileURLToPath(import.meta.url));
+  const candidates = [
+    join3(here, "../../../../crates/d4c-fs/target/release/d4c-fs"),
+    join3(here, `../../../fs-${process.platform}-${process.arch}/d4c-fs`)
+  ];
+  return candidates.find((c) => existsSync(c)) ?? null;
+}
+async function nativeDiskUsage(paths, opts = {}) {
+  const bin = findBinary(opts.binPath);
+  if (bin === null || paths.length === 0)
+    return null;
+  const args = ["disk-usage"];
+  for (const p of paths)
+    args.push("--path", p);
+  return new Promise((resolve) => {
+    execFile(bin, args, { maxBuffer: 64 * 1024 * 1024, timeout: opts.timeoutMs ?? 300000 }, (err, stdout) => {
+      if (err !== null || stdout === "") {
+        resolve(null);
+        return;
+      }
+      try {
+        resolve(JSON.parse(stdout));
+      } catch {
+        resolve(null);
+      }
+    });
+  });
+}
+var shareBin;
+async function fileShareInfoBatch(paths, opts = {}) {
+  if (paths.length === 0)
+    return new Map;
+  if (opts.binPath !== undefined)
+    shareBin = findBinary(opts.binPath);
+  else if (shareBin === undefined)
+    shareBin = findBinary();
+  if (shareBin === null)
+    return null;
+  const args = ["file-info"];
+  for (const p of paths)
+    args.push("--path", p);
+  return new Promise((resolve) => {
+    execFile(shareBin, args, { maxBuffer: 64 * 1024 * 1024, timeout: opts.timeoutMs ?? 120000 }, (err, stdout) => {
+      if (err !== null || stdout === "") {
+        resolve(null);
+        return;
+      }
+      try {
+        const j = JSON.parse(stdout);
+        resolve(new Map(j.files.map((f) => [f.path, f])));
+      } catch {
+        resolve(null);
+      }
+    });
+  });
+}
+
 // ../core/src/reap/index.ts
 function relativeTo(path, root) {
   if (path === root)
@@ -414,6 +488,21 @@ async function planReap(root, opts = {}) {
     totalBytes += c.tree.sizeBytes ?? 0;
     opts.onMeasured?.(c.tree);
   }
+  let measurementSource = "approximate";
+  let nativeGroupBytes = null;
+  if (candidates.length > 0 && !interrupted) {
+    const native = await nativeDiskUsage(candidates.map((c) => c.tree.path), { binPath: opts.nativeBinPath });
+    if (native !== null && native.measured) {
+      measurementSource = "native";
+      const byPath = new Map(native.paths.map((p) => [p.path, p]));
+      for (const c of candidates) {
+        const m = byPath.get(c.tree.path);
+        if (m !== undefined)
+          c.tree.sizeBytes = m.privateBytes;
+      }
+      nativeGroupBytes = native.group.occupancyBytes;
+    }
+  }
   const tooSmall = candidates.filter((c) => (c.tree.sizeBytes ?? 0) < minSizeBytes);
   for (const c of tooSmall)
     skipped.push({ tree: c.tree, reason: "below-min-size" });
@@ -423,12 +512,13 @@ async function planReap(root, opts = {}) {
   candidates.sort((a, b) => (b.tree.sizeBytes ?? 0) - (a.tree.sizeBytes ?? 0));
   return {
     root,
+    measurementSource,
     idleThresholdDays,
     exclude,
     minSizeBytes,
     candidates,
     skipped,
-    reclaimableBytes: candidates.reduce((s, c) => s + (c.tree.sizeBytes ?? 0), 0),
+    reclaimableBytes: nativeGroupBytes ?? candidates.reduce((s, c) => s + (c.tree.sizeBytes ?? 0), 0),
     totalBytes,
     treeCount: trees.length,
     diagnostics,
@@ -457,7 +547,7 @@ async function verifyStillReapable(c) {
     return "lockfile-vanished";
   for (const f of [tree.lockfile, "package.json"]) {
     try {
-      const st2 = await stat2(join3(tree.lockfileDir, f));
+      const st2 = await stat2(join4(tree.lockfileDir, f));
       if (!st2.isFile())
         return "lockfile-vanished";
     } catch {
@@ -581,6 +671,7 @@ async function gc(args) {
       schemaVersion: 1,
       root: args.cwd,
       idleThresholdDays: plan.idleThresholdDays,
+      measurementSource: plan.measurementSource,
       exclude: plan.exclude,
       minSizeBytes: plan.minSizeBytes,
       dryRun: result.dryRun,
@@ -666,6 +757,13 @@ async function gc(args) {
     out.push("Interrupted. The trees listed above are what was handled before stopping;");
     out.push("anything else was left untouched.");
   }
+  if (plan.measurementSource === "approximate") {
+    out.push("");
+    out.push("Sizes are approximate. Files already sharing blocks through APFS");
+    out.push("clones are counted at full size, so the figure above can be much");
+    out.push("larger than what deleting would actually free. Build crates/d4c-fs");
+    out.push("for exact numbers.");
+  }
   out.push("");
   out.push("Idle age is measured from the last install or modification, not the");
   out.push("last time the tree was read. A project you still use can look idle.");
@@ -685,11 +783,11 @@ var DEFAULT_DAYS = DEFAULT_IDLE_THRESHOLD_DAYS;
 
 // ../core/src/dedupe/index.ts
 import { lstat as lstat5, mkdir as mkdir2, appendFile as appendFile2, realpath as realpath2 } from "node:fs/promises";
-import { dirname as dirname4 } from "node:path";
+import { dirname as dirname5 } from "node:path";
 
 // ../core/src/dedupe/index-builder.ts
 import { readdir as readdir2, lstat as lstat3, open } from "node:fs/promises";
-import { join as join4 } from "node:path";
+import { join as join5 } from "node:path";
 import { createHash } from "node:crypto";
 var DEFAULT_MIN_FILE_BYTES = 65536;
 async function hashFile(path) {
@@ -727,7 +825,7 @@ async function buildDuplicateIndex(treeRoots, opts = {}) {
         continue;
       }
       for (const e of entries) {
-        const p = join4(dir, e.name);
+        const p = join5(dir, e.name);
         if (e.isDirectory()) {
           stack.push(p);
           continue;
@@ -767,7 +865,8 @@ async function buildDuplicateIndex(treeRoots, opts = {}) {
           ino: st.ino,
           mode: st.mode & 4095,
           mtimeMs: st.mtimeMs,
-          cloneRefs: 0
+          cloneRefs: 0,
+          cloneId: null
         };
         const list = byHash.get(hash) ?? [];
         list.push(entry);
@@ -775,6 +874,26 @@ async function buildDuplicateIndex(treeRoots, opts = {}) {
       }
     }
   }
+  const candidatePaths = [];
+  for (const all of byHash.values()) {
+    if (all.length >= 2)
+      for (const f of all)
+        candidatePaths.push(f.path);
+  }
+  const share = await fileShareInfoBatch(candidatePaths, { binPath: opts.nativeBinPath });
+  if (share !== null) {
+    for (const all of byHash.values()) {
+      for (const f of all) {
+        const s = share.get(f.path);
+        if (s === undefined || !s.measured)
+          continue;
+        if (s.privateBytes !== null)
+          f.sizeBytes = s.privateBytes;
+        f.cloneRefs = s.cloneRefcnt ?? 0;
+        f.cloneId = s.cloneId;
+      }
+    }
+  }
   const groups = [];
   for (const [hash, all] of byHash) {
     const byMode = new Map;
@@ -796,8 +915,11 @@ async function buildDuplicateIndex(treeRoots, opts = {}) {
         if (same.length < 2)
           continue;
         const canonical = pickCanonical(same);
-        const reclaimableBytes = same.filter((f) => f !== canonical).reduce((s, f) => s + f.sizeBytes, 0);
-        groups.push({ hash, files: same, canonical, reclaimableBytes });
+        const targets = same.filter((f) => f !== canonical && !(f.cloneId !== null && f.cloneId === canonical.cloneId));
+        if (targets.length === 0)
+          continue;
+        const reclaimableBytes = targets.reduce((s, f) => s + f.sizeBytes, 0);
+        groups.push({ hash, files: [canonical, ...targets], canonical, reclaimableBytes });
       }
     }
   }
@@ -809,11 +931,14 @@ function pickCanonical(files) {
 }
 
 // ../core/src/dedupe/replace.ts
-import { lstat as lstat4, open as open2, rm as rm2, rename, chmod, utimes } from "node:fs/promises";
-import { dirname as dirname3, join as join5 } from "node:path";
-import { randomBytes } from "node:crypto";
+import { lstat as lstat4, open as open2, rm as rm3, rename, chmod, utimes } from "node:fs/promises";
+import { dirname as dirname4, join as join7 } from "node:path";
+import { randomBytes as randomBytes2 } from "node:crypto";
 
 // ../core/src/clone/clonefile.ts
+import { rm as rm2, stat as stat3 } from "node:fs/promises";
+import { join as join6 } from "node:path";
+import { randomBytes } from "node:crypto";
 var cachedFn;
 function loadClonefile() {
   if (cachedFn !== undefined)
@@ -847,6 +972,35 @@ async function cloneFile(src, dst) {
     throw new CloneFailedError(`clonefile failed: ${src} -> ${dst}`);
   }
 }
+async function cloneSupported(dir) {
+  const tag = randomBytes(6).toString("hex");
+  const src = join6(dir, `.d4c-probe-${tag}`);
+  const dst = join6(dir, `.d4c-probe-${tag}-clone`);
+  try {
+    await Bun.write(src, "probe");
+    await cloneFile(src, dst);
+    return true;
+  } catch {
+    return false;
+  } finally {
+    await rm2(src, { force: true });
+    await rm2(dst, { force: true });
+  }
+}
+async function cloneDiagnostics(dir) {
+  if (process.platform !== "darwin") {
+    return { supported: false, reason: `clonefile is macOS-only (this is ${process.platform})` };
+  }
+  if (loadClonefile() === null) {
+    return { supported: false, reason: "koffi is not installed; run `npm i koffi` to enable dedupe" };
+  }
+  try {
+    await stat3(dir);
+  } catch {
+    return { supported: false, reason: `cannot stat ${dir}` };
+  }
+  return await cloneSupported(dir) ? { supported: true, reason: "clonefile works here" } : { supported: false, reason: "clonefile refused on this filesystem" };
+}
 
 // ../core/src/dedupe/replace.ts
 async function sameContent(a, b) {
@@ -897,11 +1051,11 @@ async function replaceWithClone(canonical, target) {
   if (!await sameContent(canonical.path, target.path)) {
     return { ok: false, reason: "content-changed" };
   }
-  const tmp = join5(dirname3(target.path), `.d4c-tmp-${randomBytes(6).toString("hex")}`);
+  const tmp = join7(dirname4(target.path), `.d4c-tmp-${randomBytes2(6).toString("hex")}`);
   try {
     await cloneFile(canonical.path, tmp);
   } catch (e) {
-    await rm2(tmp, { force: true });
+    await rm3(tmp, { force: true });
     return { ok: false, reason: "clone-failed", detail: String(e) };
   }
   try {
@@ -909,7 +1063,7 @@ async function replaceWithClone(canonical, target) {
     await utimes(tmp, ts.atime, ts.mtime);
     await rename(tmp, target.path);
   } catch (e) {
-    await rm2(tmp, { force: true });
+    await rm3(tmp, { force: true });
     return { ok: false, reason: "clone-failed", detail: String(e) };
   }
   try {
@@ -923,7 +1077,7 @@ async function replaceWithClone(canonical, target) {
 }
 
 // ../core/src/dedupe/open-files.ts
-import { execFile } from "node:child_process";
+import { execFile as execFile2 } from "node:child_process";
 import { realpath } from "node:fs/promises";
 async function openFilesUnder(roots, opts = {}) {
   if (roots.length === 0)
@@ -931,7 +1085,7 @@ async function openFilesUnder(roots, opts = {}) {
   const bin = opts.lsofPath ?? "lsof";
   const realRoots = await Promise.all(roots.map((r) => realpath(r).catch(() => r)));
   return new Promise((resolve) => {
-    execFile(bin, ["-Fn", "-w"], { maxBuffer: 64 * 1024 * 1024, timeout: opts.timeoutMs ?? 60000 }, (err, stdout) => {
+    execFile2(bin, ["-Fn", "-w"], { maxBuffer: 64 * 1024 * 1024, timeout: opts.timeoutMs ?? 60000 }, (err, stdout) => {
       if (stdout === undefined || stdout === "") {
         resolve(err ? null : new Set);
         return;
@@ -970,17 +1124,24 @@ async function planDedupe(inputRoot, opts = {}) {
       treeMtimes[t.path] = (await lstat5(t.path)).mtimeMs;
     } catch {}
   }
-  const empty = (check2) => ({
+  const empty = (check2, support = "supported", reason = "") => ({
     root,
     idleThresholdDays,
     trees,
     groups: [],
     reclaimableBytes: 0,
     openFileCheck: check2,
+    cloneSupport: support,
+    cloneReason: reason,
     excludedOpenFiles: 0,
     treeMtimes,
     interrupted: false
   });
+  const probe = opts.cloneProbe ?? cloneSupported;
+  if (!await probe(root)) {
+    const d = await cloneDiagnostics(root);
+    return empty(opts.skipOpenFileCheck === true ? "skipped" : "clean", "unsupported", d.reason);
+  }
   if (trees.length === 0)
     return empty(opts.skipOpenFileCheck === true ? "skipped" : "clean");
   let held = new Set;
@@ -1023,6 +1184,8 @@ async function planDedupe(inputRoot, opts = {}) {
     groups: filtered,
     reclaimableBytes: filtered.reduce((s, g) => s + g.reclaimableBytes, 0),
     openFileCheck: check,
+    cloneSupport: "supported",
+    cloneReason: "",
     excludedOpenFiles,
     treeMtimes,
     interrupted: opts.signal?.aborted === true
@@ -1081,7 +1244,7 @@ async function executeDedupe(plan, opts = {}) {
         freedBytes += r.freedBytes;
         if (logPath !== null) {
           try {
-            await mkdir2(dirname4(logPath), { recursive: true });
+            await mkdir2(dirname5(logPath), { recursive: true });
             await appendFile2(logPath, JSON.stringify({
               event: "dedupe",
               at: new Date().toISOString(),
@@ -1149,6 +1312,8 @@ async function dedupe(args) {
       dryRun: result.dryRun,
       interrupted: plan.interrupted || result.interrupted,
       openFileCheck: plan.openFileCheck,
+      cloneSupport: plan.cloneSupport,
+      cloneReason: plan.cloneReason,
       excludedOpenFiles: plan.excludedOpenFiles,
       treeCount: plan.trees.length,
       groupCount: plan.groups.length,
@@ -1169,6 +1334,21 @@ async function dedupe(args) {
     return 0;
   }
   const out = ["D4C — deduplicate identical dependency files", ""];
+  if (plan.cloneSupport === "unsupported") {
+    out.push("Block cloning is not available here, so nothing was examined.");
+    out.push(`  ${plan.cloneReason}`);
+    out.push("");
+    out.push("dedupe needs a filesystem that can share blocks between files:");
+    out.push("  macOS   APFS");
+    out.push("  Linux   btrfs or XFS  (ext4 cannot)");
+    out.push("  Windows ReFS, including a Dev Drive  (NTFS cannot)");
+    out.push("");
+    out.push("`d4c gc` works anywhere and does not need this.");
+    process.stdout.write(out.join(`
+`) + `
+`);
+    return 0;
+  }
   if (plan.openFileCheck === "unavailable") {
     out.push("Could not run lsof, so it is unknown whether anything has these files open.");
     out.push("Replacing a file a process is writing to would lose that write silently,");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "d4c-pool",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Pool your scattered node_modules. Deduplicate identical files across project copies, and remove trees nothing has touched.",
   "keywords": [
     "node_modules",
@@ -35,5 +35,8 @@
   },
   "dependencies": {
     "koffi": "^3.1.6"
+  },
+  "optionalDependencies": {
+    "@voidmatcha/d4c-fs-darwin-arm64": "^0.1.0"
   }
 }