@pipelab/core-node 1.0.0-beta.1 → 1.0.0-beta.11

package/src/utils/remote.ts

@@ -1,7 +1,17 @@
 import { dirname, delimiter, join } from "node:path";
 import { tmpdir } from "node:os";
-import { mkdir, readdir, readFile, writeFile, access, chmod, rm, cp } from "node:fs/promises";
-import { existsSync, constants } from "node:fs";
+import {
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+  access,
+  chmod,
+  rm,
+  cp,
+  rename,
+} from "node:fs/promises";
+import { existsSync, constants, statSync, readdirSync } from "node:fs";
 import pacote from "pacote";
 import semver from "semver";
 import { isDev, projectRoot, PipelabContext } from "../context";
@@ -12,6 +22,29 @@ import { downloadFile, extractZip, extractTarGz, generateTempFolder } from "./fs
 export const DEFAULT_NODE_VERSION = "24.14.1";
 export const DEFAULT_PNPM_VERSION = "10.12.0";
 
+function isPackageComplete(packageDir: string): boolean {
+  return existsSync(join(packageDir, "package.json"));
+}
+
+function isNodeJSComplete(nodePath: string): boolean {
+  try {
+    return existsSync(nodePath) && statSync(nodePath).size > 0;
+  } catch {
+    return false;
+  }
+}
+
+function isDependenciesInstalledSync(packageDir: string): boolean {
+  const nodeModulesPath = join(packageDir, "node_modules");
+  if (!existsSync(nodeModulesPath)) return false;
+  try {
+    const files = readdirSync(nodeModulesPath);
+    return files.length > 0;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * In-memory lock to prevent concurrent operations on the same resource (e.g., downloading Node.js).
  */
@@ -57,12 +90,16 @@ export async function fetchPackage(
   isLocal?: boolean;
   entryPoint?: string;
 }> {
+  const start = Date.now();
   // 0. Check for local monorepo package in development
   if (isDev && projectRoot && process.env.PIPELAB_FORCE_NPM !== "true") {
     if (packageName.startsWith("@pipelab/")) {
+      const localStart = Date.now();
       const local = await tryResolveMonorepoPackage(packageName);
       if (local) {
-        console.log(`[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir}`);
+        console.log(
+          `[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir} (${Date.now() - localStart}ms)`,
+        );
         return {
           ...local,
           resolvedVersion: "workspace",
@@ -76,6 +113,7 @@ export async function fetchPackage(
   let resolvedVersion: string;
 
   console.log(`[Fetcher] Resolving ${packageName}@${versionOrRange || "latest"}...`);
+  const resolveStart = Date.now();
 
   try {
     // 1. Resolve version/range using npm with session-wide memoization and disk cache
@@ -94,15 +132,25 @@ export async function fetchPackage(
     const foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
 
     if (!foundVersion) {
-      throw new Error(`Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`);
+      throw new Error(
+        `Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`,
+      );
     }
     resolvedVersion = foundVersion;
-    console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm`);
+    console.log(
+      `[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm (${Date.now() - resolveStart}ms)`,
+    );
   } catch (error) {
-    console.warn(`[Fetcher] ${packageName}: remote resolution failed, trying local fallback...`);
+    console.warn(
+      `[Fetcher] ${packageName}: remote resolution failed (${Date.now() - resolveStart}ms), trying local fallback...`,
+    );
+    const fallbackStart = Date.now();
     const fallbackVersion = await tryLocalFallback(versionOrRange, error, baseDir, packageName);
     if (fallbackVersion) {
       resolvedVersion = fallbackVersion;
+      console.log(
+        `[Fetcher] ${packageName}: Resolved to local fallback ${resolvedVersion} (${Date.now() - fallbackStart}ms)`,
+      );
     } else {
       throw error;
     }
@@ -111,28 +159,81 @@ export async function fetchPackage(
   const cachePath = join(ctx.userDataPath, "cache", "pacote");
   const packageDir = join(baseDir, resolvedVersion);
 
-  // If the package already exists and we don't need to install dependencies, return immediately
-  if (existsSync(packageDir) && !options?.installDeps) {
+  const checkStart = Date.now();
+  // If the package already exists and we don't need to install dependencies (or they are already installed), return immediately
+  const isInstalled = options?.installDeps
+    ? isPackageComplete(packageDir) && isDependenciesInstalledSync(packageDir)
+    : isPackageComplete(packageDir);
+  const checkDuration = Date.now() - checkStart;
+
+  if (isInstalled) {
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: Already installed (check took ${checkDuration}ms, fetchPackage took ${Date.now() - start}ms)`,
+    );
     return { packageDir, resolvedVersion };
   }
 
   const lockKey = `package:${packageName}:${resolvedVersion}`;
   return withLock(lockKey, async () => {
-    if (!existsSync(packageDir)) {
+    if (!isPackageComplete(packageDir)) {
       console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Downloading to ${packageDir}...`);
-      await mkdir(packageDir, { recursive: true });
-      await pacote.extract(`${packageName}@${resolvedVersion}`, packageDir, {
-        cache: cachePath,
-      });
+      const downloadStart = Date.now();
+
+      const tempDir = join(
+        baseDir,
+        `.tmp-${resolvedVersion}-${Math.random().toString(36).slice(2)}`,
+      );
+      await mkdir(tempDir, { recursive: true });
+      try {
+        await pacote.extract(`${packageName}@${resolvedVersion}`, tempDir, {
+          cache: cachePath,
+        });
+
+        // Resolve entryPoint from package.json inside the temp directory first to ensure everything works
+        await resolveEntryPoint(tempDir, packageName);
+
+        // Remove existing incomplete packageDir if any
+        if (existsSync(packageDir)) {
+          await rm(packageDir, { recursive: true, force: true }).catch(() => {});
+        }
+
+        // Atomically rename the temp directory to the target packageDir
+        try {
+          await rename(tempDir, packageDir);
+        } catch (err: any) {
+          if (isPackageComplete(packageDir)) {
+            console.log(`[Fetcher] Destination ${packageDir} already exists and is valid.`);
+          } else {
+            throw err;
+          }
+        }
+        console.log(
+          `[Fetcher] ${packageName}@${resolvedVersion}: Downloaded and extracted in ${Date.now() - downloadStart}ms`,
+        );
+      } catch (err) {
+        await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+        throw err;
+      }
     }
 
     // 2. Resolve entry point from package.json for downloaded package
+    const entryStart = Date.now();
     const entryPoint = await resolveEntryPoint(packageDir, packageName);
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: Resolved entry point in ${Date.now() - entryStart}ms`,
+    );
 
     if (options?.installDeps) {
+      const depsStart = Date.now();
       await installDependencies(packageDir, packageName, options);
+      console.log(
+        `[Fetcher] ${packageName}@${resolvedVersion}: Installed dependencies in ${Date.now() - depsStart}ms`,
+      );
     }
 
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: FetchPackage complete in ${Date.now() - start}ms`,
+    );
     return { packageDir, resolvedVersion, entryPoint };
   });
 }
@@ -150,7 +251,13 @@ export async function runPnpm(
   },
 ) {
   const {
-    args = ["install", "--prod", "--no-lockfile", "--prefer-offline", "--no-verify-store-integrity"],
+    args = [
+      "install",
+      "--prod",
+      "--no-lockfile",
+      "--prefer-offline",
+      "--no-verify-store-integrity",
+    ],
     extraEnv = {},
     signal,
     context: ctx,
@@ -183,21 +290,22 @@ export async function runPnpm(
 /**
  * Installs a specific version of Node.js if not already present.
  */
-export async function ensureNodeJS(
-  context: PipelabContext,
-  version = DEFAULT_NODE_VERSION,
-) {
+export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NODE_VERSION) {
+  const checkStart = Date.now();
   const isWindows = process.platform === "win32";
   const nodeDir = context.getThirdPartyPath("node", version);
   const finalNodePath = join(nodeDir, isWindows ? "node.exe" : "bin/node");
 
-  if (existsSync(finalNodePath)) {
+  if (isNodeJSComplete(finalNodePath)) {
+    console.log(
+      `[Environment] Node.js check took ${Date.now() - checkStart}ms (found at ${finalNodePath})`,
+    );
     return finalNodePath;
   }
 
   const lockKey = `node:${version}`;
   return withLock(lockKey, async () => {
-    if (existsSync(finalNodePath)) return finalNodePath;
+    if (isNodeJSComplete(finalNodePath)) return finalNodePath;
 
     const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : "x86";
     const platform = isWindows ? "win" : process.platform === "darwin" ? "osx" : "linux";
@@ -211,10 +319,13 @@ export async function ensureNodeJS(
 
     sendStartupProgress(`Downloading Node.js v${version}...`);
     console.log(`Downloading Node.js from ${downloadUrl}...`);
+    const dlStart = Date.now();
     await downloadFile(downloadUrl, archivePath);
+    console.log(`[Environment] Node.js download took ${Date.now() - dlStart}ms`);
 
     sendStartupProgress(`Extracting Node.js v${version}...`);
     console.log(`Extracting Node.js to ${tempDir}...`);
+    const extStart = Date.now();
     const extractTempDir = join(tempDir, "extracted");
     await mkdir(extractTempDir, { recursive: true });
 
@@ -229,12 +340,42 @@ export async function ensureNodeJS(
     if (!nodeSubDir) throw new Error(`Could not find extracted Node.js directory`);
 
     const sourceDir = join(extractTempDir, nodeSubDir);
-    await mkdir(dirname(nodeDir), { recursive: true });
-    await rm(nodeDir, { recursive: true, force: true });
-    await cp(sourceDir, nodeDir, { recursive: true });
-    await rm(tempDir, { recursive: true, force: true });
+    const parentDir = dirname(nodeDir);
+    await mkdir(parentDir, { recursive: true });
+
+    const tempNodeDir = join(
+      parentDir,
+      `.tmp-node-${version}-${Math.random().toString(36).slice(2)}`,
+    );
+    await mkdir(tempNodeDir, { recursive: true });
+
+    try {
+      await cp(sourceDir, tempNodeDir, { recursive: true });
+      if (!isWindows) {
+        const tempNodePath = join(tempNodeDir, "bin/node");
+        await chmod(tempNodePath, 0o755).catch(() => {});
+      }
+
+      if (existsSync(nodeDir)) {
+        await rm(nodeDir, { recursive: true, force: true }).catch(() => {});
+      }
 
-    if (!isWindows) await chmod(finalNodePath, 0o755).catch(() => { });
+      try {
+        await rename(tempNodeDir, nodeDir);
+      } catch (err: any) {
+        if (isNodeJSComplete(finalNodePath)) {
+          console.log(`[Fetcher] Node.js directory already exists and is valid.`);
+        } else {
+          throw err;
+        }
+      }
+      console.log(`[Environment] Node.js extraction took ${Date.now() - extStart}ms`);
+    } finally {
+      await rm(tempNodeDir, { recursive: true, force: true }).catch(() => {});
+      await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+    }
+
+    console.log(`[Environment] Node.js set up complete in ${Date.now() - checkStart}ms`);
     return finalNodePath;
   });
 }
@@ -242,14 +383,15 @@ export async function ensureNodeJS(
 /**
  * Installs the PNPM package from npm if not already present.
  */
-export async function ensurePNPM(
-  context: PipelabContext,
-  version = DEFAULT_PNPM_VERSION,
-) {
+export async function ensurePNPM(context: PipelabContext, version = DEFAULT_PNPM_VERSION) {
+  const checkStart = Date.now();
   const pnpmDir = context.getPackagesPath("pnpm", version);
   const pnpmPath = join(pnpmDir, "bin", "pnpm.cjs");
 
   if (existsSync(pnpmPath)) {
+    console.log(
+      `[Environment] PNPM check took ${Date.now() - checkStart}ms (found at ${pnpmPath})`,
+    );
     return pnpmPath;
   }
 
@@ -260,39 +402,56 @@ export async function ensurePNPM(
     const { packageDir } = await fetchPackage("pnpm", version, {
       context,
     });
+    console.log(`[Environment] PNPM set up complete in ${Date.now() - checkStart}ms`);
     return join(packageDir, "bin", "pnpm.cjs");
   });
 }
 
 async function installDependencies(packageDir: string, packageName: string, options: FetchOptions) {
+  const start = Date.now();
   const nodeModulesPath = join(packageDir, "node_modules");
 
-  if (existsSync(nodeModulesPath)) {
-    try {
-      const files = await readdir(nodeModulesPath);
-      if (files.length === 0) {
-        console.warn(`[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`);
-      }
-    } catch (e) {
-      // Continue to install if readdir fails
-    }
+  if (isDependenciesInstalledSync(packageDir)) {
+    console.log(`[Fetcher] ${packageName}: Dependencies already installed, skipping.`);
+    return;
   }
 
-  console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+  // To prevent other processes from seeing a partially populated node_modules,
+  // we can create a temp directory next to packageDir, run pnpm there, and rename node_modules
+  const tempDir = `${packageDir}.tmp-deps-${Math.random().toString(36).slice(2)}`;
+  await mkdir(tempDir, { recursive: true });
   try {
-    const { all } = await runPnpm(packageDir, {
+    // Copy package.json to tempDir so pnpm can install dependencies
+    await cp(join(packageDir, "package.json"), join(tempDir, "package.json"));
+
+    console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+    const pnpmStart = Date.now();
+    const { all } = await runPnpm(tempDir, {
       signal: options.signal,
       context: options.context,
     });
+    console.log(`[Fetcher] ${packageName}: pnpm install command took ${Date.now() - pnpmStart}ms`);
 
     if (all) console.log(`[Fetcher] ${packageName}: Installation trace:\n${all}`);
-    console.log(`[Fetcher] ${packageName}: Dependencies installed successfully.`);
+
+    // Atomically rename node_modules from tempDir to packageDir/node_modules
+    const tempNodeModules = join(tempDir, "node_modules");
+    if (existsSync(nodeModulesPath)) {
+      await rm(nodeModulesPath, { recursive: true, force: true }).catch(() => {});
+    }
+    const renameStart = Date.now();
+    await rename(tempNodeModules, nodeModulesPath);
+    console.log(
+      `[Fetcher] ${packageName}: Dependencies installed successfully (rename took ${Date.now() - renameStart}ms, total installDependencies took ${Date.now() - start}ms).`,
+    );
   } catch (err: any) {
     console.error(
       `[Fetcher] ${packageName}: CRITICAL ERROR during dependency installation: ${err.message}`,
     );
     if (err.all) console.error(`[Fetcher] ${packageName}: Error details:\n${err.all}`);
     throw new Error(`Failed to install dependencies for ${packageName}. See logs for details.`);
+  } finally {
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
   }
 }
 
@@ -316,17 +475,14 @@ export async function fetchPipelabPlugin(
   options: FetchOptions,
 ): Promise<{ packageDir: string; entryPoint: string; isLocal: boolean }> {
   const { packageDir, isLocal, entryPoint } = await fetchPackage(pluginName, versionOrRange, {
-    installDeps: true,
+    installDeps: false,
     ...options,
   });
 
   // Default entry point if not provided by fetchPackage
   let finalEntryPoint = entryPoint;
   if (!finalEntryPoint) {
-    const patterns = [
-      join(packageDir, "dist", "index.mjs"),
-      join(packageDir, "index.mjs"),
-    ];
+    const patterns = [join(packageDir, "dist", "index.mjs"), join(packageDir, "index.mjs")];
     finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
   }
 
@@ -346,10 +502,7 @@ export async function fetchPipelabCli(
   // Default entry point for CLI if not provided
   let finalEntryPoint = entryPoint;
   if (!finalEntryPoint) {
-    const patterns = [
-      join(packageDir, "dist", "index.mjs"),
-      join(packageDir, "index.mjs"),
-    ];
+    const patterns = [join(packageDir, "dist", "index.mjs"), join(packageDir, "index.mjs")];
     finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
   }
 
@@ -387,7 +540,7 @@ async function tryResolveMonorepoPackage(
       typeof pkg.bin === "string"
         ? pkg.bin
         : pkg.bin?.[packageName.replace("@pipelab/", "")] ||
-        (pkg.bin ? pkg.bin[Object.keys(pkg.bin)[0]] : undefined);
+          (pkg.bin ? pkg.bin[Object.keys(pkg.bin)[0]] : undefined);
 
     // In dev, we prefer the "main" field if it points to TS, or a hardcoded src/index.ts
     const tsSource = join(packageDir, "src", "index.ts");
@@ -434,11 +587,11 @@ async function crawlMonorepoPackages(): Promise<Record<string, string>> {
               if (pkg.name) {
                 cache[pkg.name] = join(fullDir, entry.name);
               }
-            } catch (e) { }
+            } catch (e) {}
           }
         }
       }
-    } catch (e) { }
+    } catch (e) {}
   }
   return cache;
 }

package/src/utils.ts

@@ -256,7 +256,7 @@ export const executeGraphWithHistory = async ({
       // Don't await, let it run in the background
       buildHistoryStorage.applyRetentionPolicy();
     }
-    
+
     if (shouldCleanup) {
       try {
         await rm(sandboxPath, { recursive: true, force: true });