@pipelab/core-node 1.0.0-beta.2 → 1.0.0-beta.21

package/src/utils/remote.ts

@@ -1,16 +1,48 @@
 import { dirname, delimiter, join } from "node:path";
-import { tmpdir } from "node:os";
-import { mkdir, readdir, readFile, writeFile, access, chmod, rm, cp } from "node:fs/promises";
-import { existsSync, constants } from "node:fs";
+import {
+  mkdir,
+  readdir,
+  readFile,
+  writeFile,
+  access,
+  chmod,
+  rm,
+  cp,
+  rename,
+} from "node:fs/promises";
+import { existsSync, constants, statSync, readdirSync } from "node:fs";
+import dns from "node:dns/promises";
 import pacote from "pacote";
 import semver from "semver";
-import { isDev, projectRoot, PipelabContext } from "../context";
+import { isDev, projectRoot, PipelabContext, CacheFolder } from "../context";
 import { execa } from "execa";
 import { sendStartupProgress } from "../server";
-import { downloadFile, extractZip, extractTarGz, generateTempFolder } from "./fs-extras";
+import { downloadFile, extractZip, extractTarGz } from "./fs-extras";
 
-export const DEFAULT_NODE_VERSION = "24.14.1";
-export const DEFAULT_PNPM_VERSION = "10.12.0";
+import { DEFAULT_NODE_VERSION, DEFAULT_PNPM_VERSION } from "@pipelab/constants";
+
+function isPackageComplete(packageDir: string): boolean {
+  return existsSync(join(packageDir, "package.json"));
+}
+
+function isNodeJSComplete(nodePath: string): boolean {
+  try {
+    return existsSync(nodePath) && statSync(nodePath).size > 0;
+  } catch {
+    return false;
+  }
+}
+
+function isDependenciesInstalledSync(packageDir: string): boolean {
+  const nodeModulesPath = join(packageDir, "node_modules");
+  if (!existsSync(nodeModulesPath)) return false;
+  try {
+    const files = readdirSync(nodeModulesPath);
+    return files.length > 0;
+  } catch {
+    return false;
+  }
+}
 
 /**
  * In-memory lock to prevent concurrent operations on the same resource (e.g., downloading Node.js).
@@ -37,6 +69,27 @@ async function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
   return promise;
 }
 
+let isOnlineCached: boolean | null = null;
+let lastCheckTime = 0;
+
+export async function isOnline(): Promise<boolean> {
+  const now = Date.now();
+  if (isOnlineCached !== null && now - lastCheckTime < 10000) {
+    return isOnlineCached;
+  }
+  try {
+    await Promise.race([
+      dns.lookup("registry.npmjs.org"),
+      new Promise((_, reject) => setTimeout(() => reject(new Error("timeout")), 1500)),
+    ]);
+    isOnlineCached = true;
+  } catch {
+    isOnlineCached = false;
+  }
+  lastCheckTime = now;
+  return isOnlineCached;
+}
+
 export type FetchOptions = {
   installDeps?: boolean;
   signal?: AbortSignal;
@@ -57,12 +110,16 @@ export async function fetchPackage(
   isLocal?: boolean;
   entryPoint?: string;
 }> {
+  const start = Date.now();
   // 0. Check for local monorepo package in development
   if (isDev && projectRoot && process.env.PIPELAB_FORCE_NPM !== "true") {
     if (packageName.startsWith("@pipelab/")) {
+      const localStart = Date.now();
       const local = await tryResolveMonorepoPackage(packageName);
       if (local) {
-        console.log(`[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir}`);
+        console.log(
+          `[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir} (${Date.now() - localStart}ms)`,
+        );
         return {
           ...local,
           resolvedVersion: "workspace",
@@ -75,66 +132,182 @@ export async function fetchPackage(
   const baseDir = ctx.getPackagesPath(packageName);
   let resolvedVersion: string;
 
-  console.log(`[Fetcher] Resolving ${packageName}@${versionOrRange || "latest"}...`);
-
-  try {
-    // 1. Resolve version/range using npm with session-wide memoization and disk cache
-    const cachePath = join(ctx.userDataPath, "cache", "pacote");
-    let packumentPromise = packumentRequests.get(packageName);
-    if (!packumentPromise) {
-      packumentPromise = pacote.packument(packageName, { cache: cachePath });
-      packumentRequests.set(packageName, packumentPromise);
-    }
-
-    const packument = await packumentPromise;
-    const versions = Object.keys(packument.versions);
-    const range = versionOrRange || "latest";
+  let resolvedVersionOrRange = versionOrRange;
+  if (resolvedVersionOrRange === "local") {
+    resolvedVersionOrRange = "latest";
+  }
 
-    // Prioritize tags (like 'latest', 'beta', etc.) over semver ranges
-    const foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
+  console.log(`[Fetcher] Resolving ${packageName}@${resolvedVersionOrRange || "latest"}...`);
+  const resolveStart = Date.now();
 
-    if (!foundVersion) {
-      throw new Error(
-        `Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`,
-      );
-    }
-    resolvedVersion = foundVersion;
-    console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm`);
-  } catch (error) {
-    console.warn(`[Fetcher] ${packageName}: remote resolution failed, trying local fallback...`);
-    const fallbackVersion = await tryLocalFallback(versionOrRange, error, baseDir, packageName);
+  const online = await isOnline();
+  if (!online) {
+    console.warn(
+      `[Fetcher] ${packageName}: offline mode detected (${Date.now() - resolveStart}ms), trying local fallback...`,
+    );
+    const fallbackStart = Date.now();
+    const fallbackVersion = await tryLocalFallback(
+      resolvedVersionOrRange,
+      new Error("Offline"),
+      baseDir,
+      packageName,
+    );
     if (fallbackVersion) {
       resolvedVersion = fallbackVersion;
+      console.log(
+        `[Fetcher] ${packageName}: Resolved to local fallback ${resolvedVersion} (${Date.now() - fallbackStart}ms)`,
+      );
     } else {
-      throw error;
+      throw new Error(`Offline and no local fallback version available for ${packageName}`);
+    }
+  } else {
+    try {
+      // 1. Resolve version/range using npm with session-wide memoization and disk cache
+      const cachePath = ctx.getCachePath(CacheFolder.Pacote);
+      let packumentPromise = packumentRequests.get(packageName);
+      if (!packumentPromise) {
+        packumentPromise = pacote.packument(packageName, { cache: cachePath });
+        packumentRequests.set(packageName, packumentPromise);
+      }
+
+      const packument = await packumentPromise;
+      const versions = Object.keys(packument.versions);
+      const range = resolvedVersionOrRange || "latest";
+
+      // Prioritize tags (like 'latest', 'beta', etc.) over semver ranges
+      let foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
+
+      // If we are in a non-latest releaseTag channel (like "beta" or "dev"),
+      // and we are resolving "latest", we prefer the releaseTag version if it exists
+      // and is newer than (or equal to) the latest stable version.
+      if (range === "latest" && ctx.releaseTag && ctx.releaseTag !== "latest") {
+        const releaseTagVersion = packument["dist-tags"]?.[ctx.releaseTag];
+        if (releaseTagVersion && semver.valid(releaseTagVersion)) {
+          if (
+            !foundVersion ||
+            (semver.valid(foundVersion) && semver.gte(releaseTagVersion, foundVersion))
+          ) {
+            console.log(
+              `[Fetcher] Using release tag "${ctx.releaseTag}" (${releaseTagVersion}) instead of "latest" (${foundVersion || "none"}) for ${packageName}`,
+            );
+            foundVersion = releaseTagVersion;
+          } else if (foundVersion) {
+            console.warn(
+              `[Fetcher] Tag "${ctx.releaseTag}" (${releaseTagVersion}) is older than "latest" (${foundVersion}) for ${packageName}, keeping "latest"`,
+            );
+          }
+        }
+      }
+
+      if (!foundVersion) {
+        throw new Error(
+          `Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`,
+        );
+      }
+      resolvedVersion = foundVersion;
+      console.log(
+        `[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm (${Date.now() - resolveStart}ms)`,
+      );
+    } catch (error) {
+      console.warn(
+        `[Fetcher] ${packageName}: remote resolution failed (${Date.now() - resolveStart}ms), trying local fallback...`,
+      );
+      const fallbackStart = Date.now();
+      const fallbackVersion = await tryLocalFallback(
+        resolvedVersionOrRange,
+        error,
+        baseDir,
+        packageName,
+      );
+      if (fallbackVersion) {
+        resolvedVersion = fallbackVersion;
+        console.log(
+          `[Fetcher] ${packageName}: Resolved to local fallback ${resolvedVersion} (${Date.now() - fallbackStart}ms)`,
+        );
+      } else {
+        throw error;
+      }
     }
   }
 
-  const cachePath = join(ctx.userDataPath, "cache", "pacote");
+  const cachePath = ctx.getCachePath(CacheFolder.Pacote);
   const packageDir = join(baseDir, resolvedVersion);
 
-  // If the package already exists and we don't need to install dependencies, return immediately
-  if (existsSync(packageDir) && !options?.installDeps) {
+  const checkStart = Date.now();
+  // If the package already exists and we don't need to install dependencies (or they are already installed), return immediately
+  const isInstalled = options?.installDeps
+    ? isPackageComplete(packageDir) && isDependenciesInstalledSync(packageDir)
+    : isPackageComplete(packageDir);
+  const checkDuration = Date.now() - checkStart;
+
+  if (isInstalled) {
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: Already installed (check took ${checkDuration}ms, fetchPackage took ${Date.now() - start}ms)`,
+    );
     return { packageDir, resolvedVersion };
   }
 
   const lockKey = `package:${packageName}:${resolvedVersion}`;
   return withLock(lockKey, async () => {
-    if (!existsSync(packageDir)) {
+    if (!isPackageComplete(packageDir)) {
       console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Downloading to ${packageDir}...`);
-      await mkdir(packageDir, { recursive: true });
-      await pacote.extract(`${packageName}@${resolvedVersion}`, packageDir, {
-        cache: cachePath,
-      });
+      const downloadStart = Date.now();
+
+      const tempDir = join(
+        baseDir,
+        `.tmp-${resolvedVersion}-${Math.random().toString(36).slice(2)}`,
+      );
+      await mkdir(tempDir, { recursive: true });
+      try {
+        await pacote.extract(`${packageName}@${resolvedVersion}`, tempDir, {
+          cache: cachePath,
+        });
+
+        // Resolve entryPoint from package.json inside the temp directory first to ensure everything works
+        await resolveEntryPoint(tempDir, packageName);
+
+        // Remove existing incomplete packageDir if any
+        if (existsSync(packageDir)) {
+          await rm(packageDir, { recursive: true, force: true }).catch(() => {});
+        }
+
+        // Atomically rename the temp directory to the target packageDir
+        try {
+          await rename(tempDir, packageDir);
+        } catch (err: any) {
+          if (isPackageComplete(packageDir)) {
+            console.log(`[Fetcher] Destination ${packageDir} already exists and is valid.`);
+          } else {
+            throw err;
+          }
+        }
+        console.log(
+          `[Fetcher] ${packageName}@${resolvedVersion}: Downloaded and extracted in ${Date.now() - downloadStart}ms`,
+        );
+      } catch (err) {
+        await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+        throw err;
+      }
     }
 
     // 2. Resolve entry point from package.json for downloaded package
+    const entryStart = Date.now();
     const entryPoint = await resolveEntryPoint(packageDir, packageName);
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: Resolved entry point in ${Date.now() - entryStart}ms`,
+    );
 
     if (options?.installDeps) {
+      const depsStart = Date.now();
       await installDependencies(packageDir, packageName, options);
+      console.log(
+        `[Fetcher] ${packageName}@${resolvedVersion}: Installed dependencies in ${Date.now() - depsStart}ms`,
+      );
     }
 
+    console.log(
+      `[Fetcher] ${packageName}@${resolvedVersion}: FetchPackage complete in ${Date.now() - start}ms`,
+    );
     return { packageDir, resolvedVersion, entryPoint };
   });
 }
@@ -181,7 +354,7 @@ export async function runPnpm(
       ...process.env,
       NODE_ENV: "production",
       PATH: nodePath ? `${dirname(nodePath)}${delimiter}${process.env.PATH}` : process.env.PATH,
-      PNPM_HOME: join(ctx.userDataPath, "pnpm"),
+      PNPM_HOME: ctx.getPnpmPath(),
       PNPM_ONLY_ALLOW_TRUSTED_DEPENDENCIES: "false",
       ...extraEnv,
     },
@@ -192,17 +365,21 @@ export async function runPnpm(
  * Installs a specific version of Node.js if not already present.
  */
 export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NODE_VERSION) {
+  const checkStart = Date.now();
   const isWindows = process.platform === "win32";
   const nodeDir = context.getThirdPartyPath("node", version);
   const finalNodePath = join(nodeDir, isWindows ? "node.exe" : "bin/node");
 
-  if (existsSync(finalNodePath)) {
+  if (isNodeJSComplete(finalNodePath)) {
+    console.log(
+      `[Environment] Node.js check took ${Date.now() - checkStart}ms (found at ${finalNodePath})`,
+    );
     return finalNodePath;
   }
 
   const lockKey = `node:${version}`;
   return withLock(lockKey, async () => {
-    if (existsSync(finalNodePath)) return finalNodePath;
+    if (isNodeJSComplete(finalNodePath)) return finalNodePath;
 
     const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : "x86";
     const platform = isWindows ? "win" : process.platform === "darwin" ? "osx" : "linux";
@@ -211,15 +388,18 @@ export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NO
 
     const fileName = `node-v${version}-${downloadPlatform}-${arch}.${extension}`;
     const downloadUrl = `https://nodejs.org/dist/v${version}/${fileName}`;
-    const tempDir = await generateTempFolder(tmpdir());
+    const tempDir = await context.createTempFolder("node-download-");
     const archivePath = join(tempDir, fileName);
 
     sendStartupProgress(`Downloading Node.js v${version}...`);
     console.log(`Downloading Node.js from ${downloadUrl}...`);
+    const dlStart = Date.now();
     await downloadFile(downloadUrl, archivePath);
+    console.log(`[Environment] Node.js download took ${Date.now() - dlStart}ms`);
 
     sendStartupProgress(`Extracting Node.js v${version}...`);
     console.log(`Extracting Node.js to ${tempDir}...`);
+    const extStart = Date.now();
     const extractTempDir = join(tempDir, "extracted");
     await mkdir(extractTempDir, { recursive: true });
 
@@ -234,12 +414,42 @@ export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NO
     if (!nodeSubDir) throw new Error(`Could not find extracted Node.js directory`);
 
     const sourceDir = join(extractTempDir, nodeSubDir);
-    await mkdir(dirname(nodeDir), { recursive: true });
-    await rm(nodeDir, { recursive: true, force: true });
-    await cp(sourceDir, nodeDir, { recursive: true });
-    await rm(tempDir, { recursive: true, force: true });
+    const parentDir = dirname(nodeDir);
+    await mkdir(parentDir, { recursive: true });
+
+    const tempNodeDir = join(
+      parentDir,
+      `.tmp-node-${version}-${Math.random().toString(36).slice(2)}`,
+    );
+    await mkdir(tempNodeDir, { recursive: true });
+
+    try {
+      await cp(sourceDir, tempNodeDir, { recursive: true });
+      if (!isWindows) {
+        const tempNodePath = join(tempNodeDir, "bin/node");
+        await chmod(tempNodePath, 0o755).catch(() => {});
+      }
 
-    if (!isWindows) await chmod(finalNodePath, 0o755).catch(() => {});
+      if (existsSync(nodeDir)) {
+        await rm(nodeDir, { recursive: true, force: true }).catch(() => {});
+      }
+
+      try {
+        await rename(tempNodeDir, nodeDir);
+      } catch (err: any) {
+        if (isNodeJSComplete(finalNodePath)) {
+          console.log(`[Fetcher] Node.js directory already exists and is valid.`);
+        } else {
+          throw err;
+        }
+      }
+      console.log(`[Environment] Node.js extraction took ${Date.now() - extStart}ms`);
+    } finally {
+      await rm(tempNodeDir, { recursive: true, force: true }).catch(() => {});
+      await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+    }
+
+    console.log(`[Environment] Node.js set up complete in ${Date.now() - checkStart}ms`);
     return finalNodePath;
   });
 }
@@ -248,10 +458,14 @@ export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NO
  * Installs the PNPM package from npm if not already present.
  */
 export async function ensurePNPM(context: PipelabContext, version = DEFAULT_PNPM_VERSION) {
+  const checkStart = Date.now();
   const pnpmDir = context.getPackagesPath("pnpm", version);
   const pnpmPath = join(pnpmDir, "bin", "pnpm.cjs");
 
   if (existsSync(pnpmPath)) {
+    console.log(
+      `[Environment] PNPM check took ${Date.now() - checkStart}ms (found at ${pnpmPath})`,
+    );
     return pnpmPath;
   }
 
@@ -262,44 +476,56 @@ export async function ensurePNPM(context: PipelabContext, version = DEFAULT_PNPM
     const { packageDir } = await fetchPackage("pnpm", version, {
       context,
     });
+    console.log(`[Environment] PNPM set up complete in ${Date.now() - checkStart}ms`);
     return join(packageDir, "bin", "pnpm.cjs");
   });
 }
 
 async function installDependencies(packageDir: string, packageName: string, options: FetchOptions) {
+  const start = Date.now();
   const nodeModulesPath = join(packageDir, "node_modules");
 
-  if (existsSync(nodeModulesPath)) {
-    try {
-      const files = await readdir(nodeModulesPath);
-      if (files.length === 0) {
-        console.warn(
-          `[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`,
-        );
-      } else {
-        console.log(`[Fetcher] ${packageName}: Dependencies already installed, skipping.`);
-        return;
-      }
-    } catch (e) {
-      // Continue to install if readdir fails
-    }
+  if (isDependenciesInstalledSync(packageDir)) {
+    console.log(`[Fetcher] ${packageName}: Dependencies already installed, skipping.`);
+    return;
   }
 
-  console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+  // To prevent other processes from seeing a partially populated node_modules,
+  // we can create a temp directory next to packageDir, run pnpm there, and rename node_modules
+  const tempDir = `${packageDir}.tmp-deps-${Math.random().toString(36).slice(2)}`;
+  await mkdir(tempDir, { recursive: true });
   try {
-    const { all } = await runPnpm(packageDir, {
+    // Copy package.json to tempDir so pnpm can install dependencies
+    await cp(join(packageDir, "package.json"), join(tempDir, "package.json"));
+
+    console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+    const pnpmStart = Date.now();
+    const { all } = await runPnpm(tempDir, {
       signal: options.signal,
       context: options.context,
     });
+    console.log(`[Fetcher] ${packageName}: pnpm install command took ${Date.now() - pnpmStart}ms`);
 
     if (all) console.log(`[Fetcher] ${packageName}: Installation trace:\n${all}`);
-    console.log(`[Fetcher] ${packageName}: Dependencies installed successfully.`);
+
+    // Atomically rename node_modules from tempDir to packageDir/node_modules
+    const tempNodeModules = join(tempDir, "node_modules");
+    if (existsSync(nodeModulesPath)) {
+      await rm(nodeModulesPath, { recursive: true, force: true }).catch(() => {});
+    }
+    const renameStart = Date.now();
+    await rename(tempNodeModules, nodeModulesPath);
+    console.log(
+      `[Fetcher] ${packageName}: Dependencies installed successfully (rename took ${Date.now() - renameStart}ms, total installDependencies took ${Date.now() - start}ms).`,
+    );
   } catch (err: any) {
     console.error(
       `[Fetcher] ${packageName}: CRITICAL ERROR during dependency installation: ${err.message}`,
     );
     if (err.all) console.error(`[Fetcher] ${packageName}: Error details:\n${err.all}`);
     throw new Error(`Failed to install dependencies for ${packageName}. See logs for details.`);
+  } finally {
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
   }
 }
 
@@ -444,30 +670,42 @@ async function crawlMonorepoPackages(): Promise<Record<string, string>> {
   return cache;
 }
 
-async function findLatestLocalVersion(baseDir: string): Promise<string | null> {
+async function tryLocalFallback(
+  versionOrRange: string | undefined,
+  _error: unknown,
+  baseDir: string,
+  logPrefix: string,
+): Promise<string | null> {
   if (!existsSync(baseDir)) return null;
   try {
     const entries = await readdir(baseDir, { withFileTypes: true });
-    const versions = entries
+    const localVersions = entries
       .filter((e) => e.isDirectory() || e.isSymbolicLink())
       .map((e) => e.name)
-      .sort((a, b) => b.localeCompare(a, undefined, { numeric: true, sensitivity: "base" }));
-    return versions[0] || null;
-  } catch {
-    return null;
-  }
-}
+      .filter((name) => !!semver.valid(name));
 
-async function tryLocalFallback(
-  _version: string | undefined,
-  _error: unknown,
-  baseDir: string,
-  logPrefix: string,
-): Promise<string | null> {
-  const latestLocal = await findLatestLocalVersion(baseDir);
-  if (latestLocal) {
-    console.info(`[Fetcher] ${logPrefix}: Using locally cached version: ${latestLocal}`);
-    return latestLocal;
+    if (localVersions.length === 0) return null;
+
+    const range = versionOrRange || "latest";
+
+    if (range === "latest") {
+      const sorted = localVersions.sort((a, b) => semver.rcompare(a, b));
+      const latestLocal = sorted[0] || null;
+      if (latestLocal) {
+        console.info(`[Fetcher] ${logPrefix}: Using locally cached latest version: ${latestLocal}`);
+        return latestLocal;
+      }
+    } else {
+      const matched = semver.maxSatisfying(localVersions, range);
+      if (matched) {
+        console.info(
+          `[Fetcher] ${logPrefix}: Using locally cached matching version: ${matched} for range ${range}`,
+        );
+        return matched;
+      }
+    }
+  } catch (e) {
+    console.warn(`[Fetcher] ${logPrefix}: Error during local fallback resolution:`, e);
   }
   return null;
 }

package/src/utils/storage.ts

@@ -12,7 +12,8 @@ export class JsonFileStorage {
   private logger = useLogger().logger;
 
   constructor(fileName: string, context: PipelabContext) {
-    this.filePath = path.join(context.userDataPath, fileName);
+    this.filePath = context.getConfigPath(fileName);
+
     const dir = path.dirname(this.filePath);
     if (!fs.existsSync(dir)) {
       try {

package/src/utils.ts

@@ -1,24 +1,27 @@
 import { nanoid } from "nanoid";
-import { usePlugins } from "@pipelab/shared";
-import { RendererPluginDefinition } from "@pipelab/shared";
+import {
+  usePlugins,
+  RendererPluginDefinition,
+  processGraph,
+  useLogger,
+  BuildHistoryEntry,
+  Variable,
+  AppConfig,
+  transformUrl,
+} from "@pipelab/shared";
 import { downloadFile, DownloadHooks } from "./utils/fs-extras";
 import { access, chmod, mkdir, rm, writeFile, readdir, cp } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { isDev, projectRoot, PipelabContext } from "./context";
 import { constants, existsSync } from "node:fs";
-import { processGraph } from "@pipelab/shared";
 import { handleActionExecute } from "./handler-func";
-import { useLogger } from "@pipelab/shared";
 import { BuildHistoryStorage } from "./handlers/build-history";
-import type { BuildHistoryEntry } from "@pipelab/shared";
-import type { Variable } from "@pipelab/shared";
 
-import { ensure, generateTempFolder, extractTarGz, extractZip, zipFolder } from "./utils/fs-extras";
+import { ensure, extractTarGz, extractZip, zipFolder } from "./utils/fs-extras";
 import { fetchPipelabAsset } from "./utils/remote";
 import { loadPipelabPlugin } from "./plugins-registry";
-import { setupConfigFile } from "./config";
-import { AppConfig } from "@pipelab/shared";
+import { setupSettingsConfigFile } from "./config";
 
 export const getFinalPlugins = () => {
   const { plugins } = usePlugins();
@@ -29,13 +32,6 @@ export const getFinalPlugins = () => {
   for (const plugin of plugins.value) {
     const finalNodes = [];
 
-    const transformUrl = (url: string) => {
-      if (url.startsWith("file://")) {
-        return url.replace("file://", "media://");
-      }
-      return url;
-    };
-
     const finalIcon =
       plugin.icon?.type === "image"
         ? {
@@ -106,6 +102,7 @@ export const executeGraphWithHistory = async ({
     projectName,
     projectPath,
     pipelineId,
+    cachePath,
     startTime,
     status: "running",
     logs: [],
@@ -124,7 +121,7 @@ export const executeGraphWithHistory = async ({
   }
 
   const logs: any[] = [];
-  const sandboxPath = await generateTempFolder(cachePath);
+  const sandboxPath = await ctx.createTempFolder("pipeline-sandbox-");
   const { logger } = useLogger();
   let completedSteps = 0;
   let failedSteps = 0;
@@ -152,7 +149,7 @@ export const executeGraphWithHistory = async ({
   }
 
   logger().info(`[Sandbox] Execution sandbox created at: ${sandboxPath}`);
-  const settingsFile = await setupConfigFile<AppConfig>("settings", { context: ctx });
+  const settingsFile = await setupSettingsConfigFile(ctx);
   const config = await settingsFile.getConfig();
   const shouldCleanup = true;
 
@@ -251,18 +248,17 @@ export const executeGraphWithHistory = async ({
 
     throw error;
   } finally {
-    // Apply retention policy regardless of outcome
-    if (!shouldDisableHistory) {
-      // Don't await, let it run in the background
-      buildHistoryStorage.applyRetentionPolicy();
-    }
-
     if (shouldCleanup) {
       try {
         await rm(sandboxPath, { recursive: true, force: true });
       } catch (e) {
         console.warn(`Failed to cleanup sandbox at ${sandboxPath}:`, e);
       }
+      try {
+        await rm(cachePath, { recursive: true, force: true });
+      } catch (e) {
+        console.warn(`Failed to cleanup cache at ${cachePath}:`, e);
+      }
     }
   }
 };