@pipelab/core-node 1.0.0-beta.1 → 1.0.0-beta.11

package/dist/index.mjs

@@ -2,13 +2,13 @@ import { createRequire } from "node:module";
 import path, { delimiter, dirname, isAbsolute, join, resolve } from "node:path";
 import { homedir, platform, tmpdir } from "node:os";
 import { fileURLToPath, pathToFileURL } from "node:url";
-import fs, { createWriteStream, existsSync } from "node:fs";
+import fs, { createWriteStream, existsSync, readdirSync, statSync } from "node:fs";
 import { WebSocket, WebSocketServer as WebSocketServer$1 } from "ws";
 import http from "node:http";
 import { nanoid } from "nanoid";
 import { SubscriptionRequiredError, WebSocketError, appSettingsMigrator, configRegistry, fileRepoMigrations, isRequired, isSupabaseAvailable, isWebSocketRequestMessage, processGraph, savedFileMigrator, supabase, useLogger, usePlugins } from "@pipelab/shared";
 import { getUiDevServerMissingWarning, uiDevPort, websocketPort } from "@pipelab/constants";
-import fs$1, { access, chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, unlink, writeFile } from "node:fs/promises";
+import fs$1, { access, chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
 import { execa } from "execa";
 import tar from "tar";
 import yauzl from "yauzl";
@@ -55,14 +55,16 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
 //#region src/context.ts
 const _dirname = typeof __dirname !== "undefined" ? __dirname : typeof import.meta !== "undefined" && import.meta.url ? dirname(fileURLToPath(import.meta.url)) : process.cwd();
 const isDev = process.env.NODE_ENV === "development";
-const getDefaultUserDataPath = () => {
-	return join((() => {
+const getDefaultUserDataPath = (env) => {
+	const base = (() => {
 		switch (platform()) {
 			case "win32": return process.env.APPDATA || join(homedir(), "AppData", "Roaming");
 			case "darwin": return join(homedir(), "Library", "Application Support");
 			default: return process.env.XDG_CONFIG_HOME || join(homedir(), ".config");
 		}
-	})(), "@pipelab", isDev ? "app-dev" : "app");
+	})();
+	const mode = env ?? (isDev ? "dev" : "prod");
+	return join(base, "@pipelab", mode === "dev" ? "app-dev" : mode === "beta" ? "app-beta" : "app");
 };
 /**
 * Finds the monorepo root by looking for pnpm-workspace.yaml.
@@ -78,8 +80,10 @@ function findProjectRoot(startDir) {
 const projectRoot = findProjectRoot(_dirname);
 var PipelabContext = class {
 	userDataPath;
+	releaseTag;
 	constructor(options) {
 		this.userDataPath = options.userDataPath;
+		this.releaseTag = options.releaseTag || "latest";
 	}
 	getPackagesPath(...subpaths) {
 		return join(this.userDataPath, "packages", ...subpaths);
@@ -628,25 +632,24 @@ const setupConfigFile = async (name, options) => {
 			}
 			let json = void 0;
 			try {
-				json = await migrator.migrate(originalJson, { debug: false });
+				json = await migrator.migrate(originalJson, {
+					debug: false,
+					onStep: async (state, version) => {
+						const parsedPath = path.parse(filesPath);
+						const versionedPath = path.join(parsedPath.dir, `${parsedPath.name}.v${version}.json`);
+						try {
+							await fs$1.writeFile(versionedPath, JSON.stringify(state));
+							logger().info(`Intermediate backup created for ${name} at ${versionedPath}`);
+						} catch (e) {
+							logger().error(`Failed to create intermediate backup for ${name} at v${version}:`, e);
+						}
+					}
+				});
 			} catch (e) {
 				logger().error(`Error migrating config ${name}:`, e);
 				json = migrator.defaultValue;
 			}
-			const originalVersion = originalJson?.version;
-			const newVersion = json?.version;
-			if (originalVersion !== newVersion && content !== void 0) {
-				const parsedPath = path.parse(filesPath);
-				const versionSuffix = originalVersion || "unknown";
-				const backupPath = path.join(parsedPath.dir, `${parsedPath.name}.${versionSuffix}.bak`);
-				try {
-					await fs$1.copyFile(filesPath, backupPath);
-					logger().info(`Backup created for ${name} at ${backupPath} (version ${versionSuffix})`);
-				} catch (e) {
-					logger().error(`Failed to create backup for ${name}:`, e);
-				}
-			}
-			if (originalVersion !== newVersion || content === void 0) try {
+			if (originalJson?.version !== json?.version || content === void 0) try {
 				await fs$1.writeFile(filesPath, JSON.stringify(json));
 			} catch (e) {
 				logger().error(`Error saving migrated config ${name}:`, e);
@@ -8832,9 +8835,13 @@ const sendStartupReady = () => {
 };
 async function serveCommand(options, version, _dirname) {
 	if (!options.userData) throw new Error("userDataPath is required for serveCommand");
-	const context = new PipelabContext({ userDataPath: options.userData });
+	const releaseTag = version.includes("beta") ? "beta" : "latest";
+	const context = new PipelabContext({
+		userDataPath: options.userData,
+		releaseTag
+	});
 	let rawAssetFolder;
-	if (!isDev) rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", "latest", { context });
+	if (!isDev) rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", releaseTag, { context });
 	const server = http$1.createServer(async (request, response) => {
 		if (isDev) {
 			response.writeHead(200, { "Content-Type": "text/html" });
@@ -8874,6 +8881,25 @@ async function serveCommand(options, version, _dirname) {
 //#region src/utils/remote.ts
 const DEFAULT_NODE_VERSION = "24.14.1";
 const DEFAULT_PNPM_VERSION = "10.12.0";
+function isPackageComplete(packageDir) {
+	return existsSync(join(packageDir, "package.json"));
+}
+function isNodeJSComplete(nodePath) {
+	try {
+		return existsSync(nodePath) && statSync(nodePath).size > 0;
+	} catch {
+		return false;
+	}
+}
+function isDependenciesInstalledSync(packageDir) {
+	const nodeModulesPath = join(packageDir, "node_modules");
+	if (!existsSync(nodeModulesPath)) return false;
+	try {
+		return readdirSync(nodeModulesPath).length > 0;
+	} catch {
+		return false;
+	}
+}
 /**
 * In-memory lock to prevent concurrent operations on the same resource (e.g., downloading Node.js).
 */
@@ -8900,11 +8926,13 @@ async function withLock(key, fn) {
 * Centralized in core-node to avoid circular dependencies.
 */
 async function fetchPackage(packageName, versionOrRange, options) {
+	const start = Date.now();
 	if (isDev && projectRoot && process.env.PIPELAB_FORCE_NPM !== "true") {
 		if (packageName.startsWith("@pipelab/")) {
+			const localStart = Date.now();
 			const local = await tryResolveMonorepoPackage(packageName);
 			if (local) {
-				console.log(`[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir}`);
+				console.log(`[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir} (${Date.now() - localStart}ms)`);
 				return {
 					...local,
 					resolvedVersion: "workspace"
@@ -8916,6 +8944,7 @@ async function fetchPackage(packageName, versionOrRange, options) {
 	const baseDir = ctx.getPackagesPath(packageName);
 	let resolvedVersion;
 	console.log(`[Fetcher] Resolving ${packageName}@${versionOrRange || "latest"}...`);
+	const resolveStart = Date.now();
 	try {
 		const cachePath = join(ctx.userDataPath, "cache", "pacote");
 		let packumentPromise = packumentRequests.get(packageName);
@@ -8929,27 +8958,65 @@ async function fetchPackage(packageName, versionOrRange, options) {
 		const foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
 		if (!foundVersion) throw new Error(`Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`);
 		resolvedVersion = foundVersion;
-		console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm`);
+		console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm (${Date.now() - resolveStart}ms)`);
 	} catch (error) {
-		console.warn(`[Fetcher] ${packageName}: remote resolution failed, trying local fallback...`);
+		console.warn(`[Fetcher] ${packageName}: remote resolution failed (${Date.now() - resolveStart}ms), trying local fallback...`);
+		const fallbackStart = Date.now();
 		const fallbackVersion = await tryLocalFallback(versionOrRange, error, baseDir, packageName);
-		if (fallbackVersion) resolvedVersion = fallbackVersion;
-		else throw error;
+		if (fallbackVersion) {
+			resolvedVersion = fallbackVersion;
+			console.log(`[Fetcher] ${packageName}: Resolved to local fallback ${resolvedVersion} (${Date.now() - fallbackStart}ms)`);
+		} else throw error;
 	}
 	const cachePath = join(ctx.userDataPath, "cache", "pacote");
 	const packageDir = join(baseDir, resolvedVersion);
-	if (existsSync(packageDir) && !options?.installDeps) return {
-		packageDir,
-		resolvedVersion
-	};
+	const checkStart = Date.now();
+	const isInstalled = options?.installDeps ? isPackageComplete(packageDir) && isDependenciesInstalledSync(packageDir) : isPackageComplete(packageDir);
+	const checkDuration = Date.now() - checkStart;
+	if (isInstalled) {
+		console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Already installed (check took ${checkDuration}ms, fetchPackage took ${Date.now() - start}ms)`);
+		return {
+			packageDir,
+			resolvedVersion
+		};
+	}
 	return withLock(`package:${packageName}:${resolvedVersion}`, async () => {
-		if (!existsSync(packageDir)) {
+		if (!isPackageComplete(packageDir)) {
 			console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Downloading to ${packageDir}...`);
-			await mkdir(packageDir, { recursive: true });
-			await pacote.extract(`${packageName}@${resolvedVersion}`, packageDir, { cache: cachePath });
+			const downloadStart = Date.now();
+			const tempDir = join(baseDir, `.tmp-${resolvedVersion}-${Math.random().toString(36).slice(2)}`);
+			await mkdir(tempDir, { recursive: true });
+			try {
+				await pacote.extract(`${packageName}@${resolvedVersion}`, tempDir, { cache: cachePath });
+				await resolveEntryPoint(tempDir, packageName);
+				if (existsSync(packageDir)) await rm(packageDir, {
+					recursive: true,
+					force: true
+				}).catch(() => {});
+				try {
+					await rename(tempDir, packageDir);
+				} catch (err) {
+					if (isPackageComplete(packageDir)) console.log(`[Fetcher] Destination ${packageDir} already exists and is valid.`);
+					else throw err;
+				}
+				console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Downloaded and extracted in ${Date.now() - downloadStart}ms`);
+			} catch (err) {
+				await rm(tempDir, {
+					recursive: true,
+					force: true
+				}).catch(() => {});
+				throw err;
+			}
 		}
+		const entryStart = Date.now();
 		const entryPoint = await resolveEntryPoint(packageDir, packageName);
-		if (options?.installDeps) await installDependencies(packageDir, packageName, options);
+		console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Resolved entry point in ${Date.now() - entryStart}ms`);
+		if (options?.installDeps) {
+			const depsStart = Date.now();
+			await installDependencies(packageDir, packageName, options);
+			console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Installed dependencies in ${Date.now() - depsStart}ms`);
+		}
+		console.log(`[Fetcher] ${packageName}@${resolvedVersion}: FetchPackage complete in ${Date.now() - start}ms`);
 		return {
 			packageDir,
 			resolvedVersion,
@@ -8988,12 +9055,16 @@ async function runPnpm(cwd, options) {
 * Installs a specific version of Node.js if not already present.
 */
 async function ensureNodeJS(context, version = DEFAULT_NODE_VERSION) {
+	const checkStart = Date.now();
 	const isWindows = process.platform === "win32";
 	const nodeDir = context.getThirdPartyPath("node", version);
 	const finalNodePath = join(nodeDir, isWindows ? "node.exe" : "bin/node");
-	if (existsSync(finalNodePath)) return finalNodePath;
+	if (isNodeJSComplete(finalNodePath)) {
+		console.log(`[Environment] Node.js check took ${Date.now() - checkStart}ms (found at ${finalNodePath})`);
+		return finalNodePath;
+	}
 	return withLock(`node:${version}`, async () => {
-		if (existsSync(finalNodePath)) return finalNodePath;
+		if (isNodeJSComplete(finalNodePath)) return finalNodePath;
 		const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : "x86";
 		const platform = isWindows ? "win" : process.platform === "darwin" ? "osx" : "linux";
 		const extension = isWindows ? "zip" : "tar.gz";
@@ -9003,9 +9074,12 @@ async function ensureNodeJS(context, version = DEFAULT_NODE_VERSION) {
 		const archivePath = join(tempDir, fileName);
 		sendStartupProgress(`Downloading Node.js v${version}...`);
 		console.log(`Downloading Node.js from ${downloadUrl}...`);
+		const dlStart = Date.now();
 		await downloadFile(downloadUrl, archivePath);
+		console.log(`[Environment] Node.js download took ${Date.now() - dlStart}ms`);
 		sendStartupProgress(`Extracting Node.js v${version}...`);
 		console.log(`Extracting Node.js to ${tempDir}...`);
+		const extStart = Date.now();
 		const extractTempDir = join(tempDir, "extracted");
 		await mkdir(extractTempDir, { recursive: true });
 		if (extension === "zip") await extractZip(archivePath, extractTempDir);
@@ -9013,17 +9087,35 @@ async function ensureNodeJS(context, version = DEFAULT_NODE_VERSION) {
 		const nodeSubDir = (await readdir(extractTempDir)).find((entry) => entry.startsWith(`node-v${version}`));
 		if (!nodeSubDir) throw new Error(`Could not find extracted Node.js directory`);
 		const sourceDir = join(extractTempDir, nodeSubDir);
-		await mkdir(dirname(nodeDir), { recursive: true });
-		await rm(nodeDir, {
-			recursive: true,
-			force: true
-		});
-		await cp(sourceDir, nodeDir, { recursive: true });
-		await rm(tempDir, {
-			recursive: true,
-			force: true
-		});
-		if (!isWindows) await chmod(finalNodePath, 493).catch(() => {});
+		const parentDir = dirname(nodeDir);
+		await mkdir(parentDir, { recursive: true });
+		const tempNodeDir = join(parentDir, `.tmp-node-${version}-${Math.random().toString(36).slice(2)}`);
+		await mkdir(tempNodeDir, { recursive: true });
+		try {
+			await cp(sourceDir, tempNodeDir, { recursive: true });
+			if (!isWindows) await chmod(join(tempNodeDir, "bin/node"), 493).catch(() => {});
+			if (existsSync(nodeDir)) await rm(nodeDir, {
+				recursive: true,
+				force: true
+			}).catch(() => {});
+			try {
+				await rename(tempNodeDir, nodeDir);
+			} catch (err) {
+				if (isNodeJSComplete(finalNodePath)) console.log(`[Fetcher] Node.js directory already exists and is valid.`);
+				else throw err;
+			}
+			console.log(`[Environment] Node.js extraction took ${Date.now() - extStart}ms`);
+		} finally {
+			await rm(tempNodeDir, {
+				recursive: true,
+				force: true
+			}).catch(() => {});
+			await rm(tempDir, {
+				recursive: true,
+				force: true
+			}).catch(() => {});
+		}
+		console.log(`[Environment] Node.js set up complete in ${Date.now() - checkStart}ms`);
 		return finalNodePath;
 	});
 }
@@ -9031,32 +9123,56 @@ async function ensureNodeJS(context, version = DEFAULT_NODE_VERSION) {
 * Installs the PNPM package from npm if not already present.
 */
 async function ensurePNPM(context, version = DEFAULT_PNPM_VERSION) {
+	const checkStart = Date.now();
 	const pnpmPath = join(context.getPackagesPath("pnpm", version), "bin", "pnpm.cjs");
-	if (existsSync(pnpmPath)) return pnpmPath;
+	if (existsSync(pnpmPath)) {
+		console.log(`[Environment] PNPM check took ${Date.now() - checkStart}ms (found at ${pnpmPath})`);
+		return pnpmPath;
+	}
 	return withLock(`pnpm:${version}`, async () => {
 		if (existsSync(pnpmPath)) return pnpmPath;
 		sendStartupProgress(`Checking PNPM v${version}...`);
 		const { packageDir } = await fetchPackage("pnpm", version, { context });
+		console.log(`[Environment] PNPM set up complete in ${Date.now() - checkStart}ms`);
 		return join(packageDir, "bin", "pnpm.cjs");
 	});
 }
 async function installDependencies(packageDir, packageName, options) {
+	const start = Date.now();
 	const nodeModulesPath = join(packageDir, "node_modules");
-	if (existsSync(nodeModulesPath)) try {
-		if ((await readdir(nodeModulesPath)).length === 0) console.warn(`[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`);
-	} catch (e) {}
-	console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+	if (isDependenciesInstalledSync(packageDir)) {
+		console.log(`[Fetcher] ${packageName}: Dependencies already installed, skipping.`);
+		return;
+	}
+	const tempDir = `${packageDir}.tmp-deps-${Math.random().toString(36).slice(2)}`;
+	await mkdir(tempDir, { recursive: true });
 	try {
-		const { all } = await runPnpm(packageDir, {
+		await cp(join(packageDir, "package.json"), join(tempDir, "package.json"));
+		console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+		const pnpmStart = Date.now();
+		const { all } = await runPnpm(tempDir, {
 			signal: options.signal,
 			context: options.context
 		});
+		console.log(`[Fetcher] ${packageName}: pnpm install command took ${Date.now() - pnpmStart}ms`);
 		if (all) console.log(`[Fetcher] ${packageName}: Installation trace:\n${all}`);
-		console.log(`[Fetcher] ${packageName}: Dependencies installed successfully.`);
+		const tempNodeModules = join(tempDir, "node_modules");
+		if (existsSync(nodeModulesPath)) await rm(nodeModulesPath, {
+			recursive: true,
+			force: true
+		}).catch(() => {});
+		const renameStart = Date.now();
+		await rename(tempNodeModules, nodeModulesPath);
+		console.log(`[Fetcher] ${packageName}: Dependencies installed successfully (rename took ${Date.now() - renameStart}ms, total installDependencies took ${Date.now() - start}ms).`);
 	} catch (err) {
 		console.error(`[Fetcher] ${packageName}: CRITICAL ERROR during dependency installation: ${err.message}`);
 		if (err.all) console.error(`[Fetcher] ${packageName}: Error details:\n${err.all}`);
 		throw new Error(`Failed to install dependencies for ${packageName}. See logs for details.`);
+	} finally {
+		await rm(tempDir, {
+			recursive: true,
+			force: true
+		}).catch(() => {});
 	}
 }
 async function fetchPipelabAsset(packageName, versionOrRange, options) {
@@ -9069,7 +9185,7 @@ async function fetchPipelabAsset(packageName, versionOrRange, options) {
 }
 async function fetchPipelabPlugin(pluginName, versionOrRange, options) {
 	const { packageDir, isLocal, entryPoint } = await fetchPackage(pluginName, versionOrRange, {
-		installDeps: true,
+		installDeps: false,
 		...options
 	});
 	let finalEntryPoint = entryPoint;
@@ -9342,11 +9458,15 @@ const DEFAULT_PLUGIN_IDS = [
 	"netlify"
 ];
 const loadPipelabPlugin = async (id, options) => {
+	const start = Date.now();
 	try {
-		const { packageDir, entryPoint } = await fetchPipelabPlugin(`@pipelab/plugin-${id}`, "latest", {
+		const packageName = `@pipelab/plugin-${id}`;
+		const fetchStart = Date.now();
+		const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, options.context.releaseTag, {
 			context: options.context,
-			installDeps: true
+			installDeps: false
 		});
+		const fetchDuration = Date.now() - fetchStart;
 		console.log(`[Plugins] [${id}] Attempting to import from: ${entryPoint}`);
 		if (!existsSync(entryPoint)) {
 			console.error(`[Plugins] [${id}] CRITICAL: Plugin entry point not found at ${entryPoint}`);
@@ -9355,25 +9475,46 @@ const loadPipelabPlugin = async (id, options) => {
 				console.log(`[Plugins] [${id}] Directory contents:`, files);
 			} catch (e) {}
 		}
+		const importStart = Date.now();
 		const pluginModule = await import(pathToFileURL(entryPoint).href);
-		console.log(`[Plugins] [${id}] Successfully loaded from: ${packageDir}`);
+		const importDuration = Date.now() - importStart;
+		const totalDuration = Date.now() - start;
+		console.log(`[Plugins] [${id}] Successfully loaded from: ${packageDir} (fetch: ${fetchDuration}ms, import: ${importDuration}ms, total: ${totalDuration}ms)`);
 		return pluginModule.default;
 	} catch (e) {
-		console.error(`[Plugins] [${id}] CRITICAL: Failed to load:`, e);
+		console.error(`[Plugins] [${id}] CRITICAL: Failed to load after ${Date.now() - start}ms:`, e);
 		if (e.code === "ERR_MODULE_NOT_FOUND") console.error(`[Plugins] [${id}] This usually means a dependency is missing in the plugin's node_modules.`);
 		return null;
 	}
 };
 const builtInPlugins = async (options) => {
-	console.log("[Plugins] Finalizing default plugins list...");
+	console.log("[Plugins] Starting background plugin loading...");
 	sendStartupProgress("Preparing environment...");
+	const envStart = Date.now();
 	await Promise.all([ensureNodeJS(options.context), ensurePNPM(options.context)]);
-	const filtered = (await Promise.allSettled(DEFAULT_PLUGIN_IDS.map(async (id) => {
-		sendStartupProgress(`Loading plugin: ${id}`);
-		return loadPipelabPlugin(id, options);
-	}))).filter((r) => r.status === "fulfilled").map((r) => r.value).filter(Boolean).flat();
-	console.log(`[Plugins] Successfully loaded ${filtered.length} default plugins`);
-	return filtered;
+	console.log(`[Plugins] Environment preparation took ${Date.now() - envStart}ms`);
+	const { usePlugins } = await import("@pipelab/shared");
+	const { registerPlugins } = usePlugins();
+	const { webSocketServer } = await import("./index.mjs");
+	(async () => {
+		const totalStart = Date.now();
+		for (const id of DEFAULT_PLUGIN_IDS) {
+			sendStartupProgress(`Loading plugin: ${id}`);
+			const pluginStart = Date.now();
+			const plugin = await loadPipelabPlugin(id, options);
+			console.log(`[Plugins] [${id}] loadPipelabPlugin loop step took ${Date.now() - pluginStart}ms`);
+			if (plugin) {
+				registerPlugins([plugin]);
+				webSocketServer.broadcast("plugin:loaded", { plugin });
+			}
+		}
+		console.log(`[Plugins] All default plugins loaded in ${Date.now() - totalStart}ms.`);
+		sendStartupProgress("All plugins loaded.");
+		setTimeout(() => {
+			webSocketServer.broadcast("startup:progress", { type: "ready" });
+		}, 2e3);
+	})();
+	return [];
 };
 //#endregion
 //#region src/utils.ts
@@ -10268,7 +10409,7 @@ const registerAllHandlers = async (options) => {
 	registerAuthHandlers(context);
 	registerSystemHandlers(options);
 	const { registerPlugins } = usePlugins();
-	registerPlugins(await builtInPlugins({ context }));
+	builtInPlugins({ context });
 };
 //#endregion
 //#region ../../node_modules/klona/dist/index.mjs