@pipelab/core-node 1.0.0-beta.1 → 1.0.0-beta.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipelab/core-node",
-  "version": "1.0.0-beta.1",
+  "version": "1.0.0-beta.2",
   "private": false,
   "description": "The Pipelab automation engine for Node.js",
   "license": "FSL-1.1-MIT",
@@ -26,6 +26,7 @@
     "access": "public"
   },
   "dependencies": {
+    "@supabase/supabase-js": "2.99.3",
     "adm-zip": "0.5.16",
     "archiver": "7.0.1",
     "check-disk-space": "3.4.0",
@@ -39,9 +40,8 @@
     "type-fest": "4.39.0",
     "ws": "8.18.3",
     "yauzl": "2.10.0",
-    "@supabase/supabase-js": "2.99.3",
-    "@pipelab/constants": "1.0.0-beta.0",
-    "@pipelab/shared": "1.0.0-beta.0"
+    "@pipelab/constants": "1.0.0-beta.1",
+    "@pipelab/shared": "1.0.0-beta.1"
   },
   "devDependencies": {
     "@types/adm-zip": "0.5.7",
@@ -54,7 +54,7 @@
     "@types/yauzl": "2.10.3",
     "tsdown": "0.21.2",
     "typescript": "^5.0.0",
-    "@pipelab/tsconfig": "1.0.0-beta.0"
+    "@pipelab/tsconfig": "1.0.0-beta.1"
   },
   "scripts": {
     "build": "tsdown",

package/src/handler-func.ts

@@ -53,9 +53,9 @@ export const handleConditionExecute = async (
     .find((plugin) => plugin.id === pluginId)
     ?.nodes.find((node: any) => node.node.id === nodeId) as
     | {
-      node: Condition;
-      runner: ConditionRunner<any>;
-    }
+        node: Condition;
+        runner: ConditionRunner<any>;
+      }
     | undefined;
 
   if (!node) {
@@ -136,9 +136,9 @@ export const handleActionExecute = async (
     .find((plugin) => plugin.id === pluginId)
     ?.nodes.find((node: any) => node.node.id === nodeId) as
     | {
-      node: Action;
-      runner: ActionRunner<any>;
-    }
+        node: Action;
+        runner: ActionRunner<any>;
+      }
     | undefined;
 
   if (!node) {

package/src/handlers/build-history.ts

@@ -93,17 +93,13 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
 
         // 2. Filter by maxEntries (sort by date first to keep the newest)
         if (maxEntries > 0 && entries.length > maxEntries) {
-          entries = entries
-            .sort((a, b) => b.createdAt - a.createdAt)
-            .slice(0, maxEntries);
+          entries = entries.sort((a, b) => b.createdAt - a.createdAt).slice(0, maxEntries);
         }
 
         if (entries.length < originalCount) {
           this.logger
             .logger()
-            .info(
-              `[${pipelineId}] Pruned ${originalCount - entries.length} history entries.`,
-            );
+            .info(`[${pipelineId}] Pruned ${originalCount - entries.length} history entries.`);
           await this.savePipelineHistory(pipelineId, entries);
         }
       }
@@ -271,8 +267,10 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
       await unlink(pipelinePath);
       this.logger.logger().info(`Cleared history for pipeline "${pipelineId}"`);
     } catch (error: any) {
-      if (error.code === 'ENOENT') {
-        this.logger.logger().warn(`No history file found for pipeline "${pipelineId}". Nothing to clear.`);
+      if (error.code === "ENOENT") {
+        this.logger
+          .logger()
+          .warn(`No history file found for pipeline "${pipelineId}". Nothing to clear.`);
         return;
       }
       this.logger.logger().error(`Failed to clear history for pipeline "${pipelineId}":`, error);

package/src/handlers/history.ts

@@ -127,8 +127,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to get build history entries",
+          ipcError: error instanceof Error ? error.message : "Failed to get build history entries",
         },
       });
     }
@@ -158,8 +157,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to update build history entry",
+          ipcError: error instanceof Error ? error.message : "Failed to update build history entry",
         },
       });
     }
@@ -189,8 +187,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to delete build history entry",
+          ipcError: error instanceof Error ? error.message : "Failed to delete build history entry",
         },
       });
     }
@@ -250,7 +247,8 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError: error instanceof Error ? error.message : "Failed to clear build history for pipeline",
+          ipcError:
+            error instanceof Error ? error.message : "Failed to clear build history for pipeline",
         },
       });
     }
@@ -290,7 +288,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
   handle("build-history:configure", async (_, { send, value }) => {
     try {
       logger().info("Updating build history configuration:", value.config);
-      
+
       const settings = await setupConfigFile<AppConfig>("settings", { context });
       const currentConfig = await settings.getConfig();
 

package/src/handlers/index.ts

@@ -26,10 +26,10 @@ export const registerAllHandlers = async (options: {
   registerSystemHandlers(options);
 
   const { registerPlugins } = usePlugins();
-  const plugins = await builtInPlugins({
+  // Execute in the background! The plugins will be dynamically registered and broadcasted to the UI.
+  builtInPlugins({
     context,
   });
-  registerPlugins(plugins as any);
 };
 
 export { registerShellHandlers } from "./shell";

package/src/plugins-registry.ts

@@ -25,7 +25,7 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
     const packageName = `@pipelab/plugin-${id}`;
     const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, "latest", {
       context: options.context,
-      installDeps: true,
+      installDeps: false,
     });
 
     console.log(`[Plugins] [${id}] Attempting to import from: ${entryPoint}`);
@@ -34,7 +34,7 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
       try {
         const files = await readdir(packageDir, { recursive: true });
         console.log(`[Plugins] [${id}] Directory contents:`, files);
-      } catch (e) { }
+      } catch (e) {}
     }
 
     const pluginModule = await import(pathToFileURL(entryPoint).href);
@@ -52,27 +52,33 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
 };
 
 export const builtInPlugins = async (options: { context: PipelabContext }) => {
-  console.log("[Plugins] Finalizing default plugins list...");
+  console.log("[Plugins] Starting background plugin loading...");
 
   // Pre-ensure Node.js and PNPM once in parallel so plugins don't have to wait for them
   sendStartupProgress("Preparing environment...");
-  await Promise.all([
-    ensureNodeJS(options.context),
-    ensurePNPM(options.context),
-  ]);
+  await Promise.all([ensureNodeJS(options.context), ensurePNPM(options.context)]);
 
-  const results = await Promise.allSettled(
+  const { usePlugins } = await import("@pipelab/shared");
+  const { registerPlugins } = usePlugins();
+  const { webSocketServer } = await import("./index");
+
+  // Load plugins asynchronously in the background
+  Promise.allSettled(
     DEFAULT_PLUGIN_IDS.map(async (id) => {
       sendStartupProgress(`Loading plugin: ${id}`);
-      return loadPipelabPlugin(id, options);
+      const plugin = await loadPipelabPlugin(id, options);
+      if (plugin) {
+        registerPlugins([plugin]);
+        webSocketServer.broadcast("plugin:loaded", { plugin });
+      }
     }),
-  );
-
-  const plugins = results
-    .filter((r) => r.status === "fulfilled")
-    .map((r) => (r as PromiseFulfilledResult<any>).value);
+  ).then(() => {
+    console.log("[Plugins] All default plugins loaded.");
+    sendStartupProgress("All plugins loaded.");
+    setTimeout(() => {
+      webSocketServer.broadcast("startup:progress", { type: "done" });
+    }, 2000);
+  });
 
-  const filtered = plugins.filter(Boolean).flat();
-  console.log(`[Plugins] Successfully loaded ${filtered.length} default plugins`);
-  return filtered;
+  return [];
 };

package/src/runner.ts

@@ -79,10 +79,7 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
   const cloudRunId = process.env.CLOUD_RUN_ID;
   if (options.cloud && cloudRunId) {
     const { createClient } = await import("@supabase/supabase-js");
-    supabase = createClient(
-      process.env.SUPABASE_URL!,
-      process.env.SUPABASE_SERVICE_ROLE_KEY!
-    );
+    supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!);
     console.log(`Cloud mode enabled. Streaming logs for run: ${cloudRunId}`);
   }
 
@@ -97,23 +94,29 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
       onNodeEnter: (node) => {
         console.log(`[ENTER] ${node.name} (${node.uid})`);
         if (supabase) {
-          supabase.from("cloud_run_logs").insert({
-            run_id: cloudRunId,
-            message: `[ENTER] ${node.name} (${node.uid})`,
-            node_uid: node.uid,
-            type: "node-enter"
-          }).then();
+          supabase
+            .from("cloud_run_logs")
+            .insert({
+              run_id: cloudRunId,
+              message: `[ENTER] ${node.name} (${node.uid})`,
+              node_uid: node.uid,
+              type: "node-enter",
+            })
+            .then();
         }
       },
       onNodeExit: (node) => {
         console.log(`[EXIT] ${node.name} (${node.uid})`);
         if (supabase) {
-          supabase.from("cloud_run_logs").insert({
-            run_id: cloudRunId,
-            message: `[EXIT] ${node.name} (${node.uid})`,
-            node_uid: node.uid,
-            type: "node-exit"
-          }).then();
+          supabase
+            .from("cloud_run_logs")
+            .insert({
+              run_id: cloudRunId,
+              message: `[EXIT] ${node.name} (${node.uid})`,
+              node_uid: node.uid,
+              type: "node-exit",
+            })
+            .then();
         }
       },
       onLog: (data, node) => {
@@ -121,12 +124,15 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
           const message = data.data.message.join(" ");
           console.log(`[LOG] ${message}`);
           if (supabase) {
-            supabase.from("cloud_run_logs").insert({
-              run_id: cloudRunId,
-              message: message,
-              node_uid: node?.uid,
-              type: "log"
-            }).then();
+            supabase
+              .from("cloud_run_logs")
+              .insert({
+                run_id: cloudRunId,
+                message: message,
+                node_uid: node?.uid,
+                type: "log",
+              })
+              .then();
           }
         }
       },

package/src/server.ts

@@ -66,7 +66,7 @@ export async function serveCommand(options: ServeOptions, version: string, _dirn
       response.writeHead(404, { "Content-Type": "text/plain" });
       response.end(
         `Error: UI directory not found at ${rawAssetFolder}.\n` +
-        "Please run 'pnpm build' in apps/ui to generate the distribution.",
+          "Please run 'pnpm build' in apps/ui to generate the distribution.",
       );
       return;
     }

package/src/utils/fs-extras.ts

@@ -1,6 +1,15 @@
 import { mkdir, createWriteStream, createReadStream } from "node:fs";
 import { execa, Options, Subprocess } from "execa";
-import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod, stat, readdir } from "node:fs/promises";
+import {
+  mkdir as mkdirP,
+  access,
+  writeFile,
+  realpath,
+  mkdtemp,
+  chmod,
+  stat,
+  readdir,
+} from "node:fs/promises";
 import { join, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import tar from "tar";

package/src/utils/github.ts

@@ -22,7 +22,7 @@ export interface FetchReleaseOptions {
  */
 export async function fetchPackageReleases(
   packageName: string,
-  options: FetchReleaseOptions = {}
+  options: FetchReleaseOptions = {},
 ): Promise<GitHubRelease[]> {
   const { repo = "CynToolkit/pipelab", allowPrerelease = false } = options;
   const url = `https://api.github.com/repos/${repo}/releases`;
@@ -61,7 +61,7 @@ export async function fetchPackageReleases(
  */
 export async function fetchLatestPackageRelease(
   packageName: string,
-  options: FetchReleaseOptions = {}
+  options: FetchReleaseOptions = {},
 ): Promise<GitHubRelease | null> {
   const releases = await fetchPackageReleases(packageName, options);
 
@@ -75,7 +75,9 @@ export async function fetchLatestPackageRelease(
     const targetTag = override.includes("@") ? override : `${packageName}@${override}`;
     const release = releases.find((r) => r.tag_name === targetTag);
     if (release) {
-      console.log(`[GitHub] Using release override from PIPELAB_OVERRIDE_RELEASE: ${release.tag_name}`);
+      console.log(
+        `[GitHub] Using release override from PIPELAB_OVERRIDE_RELEASE: ${release.tag_name}`,
+      );
       return release;
     }
     console.warn(`[GitHub] Override release ${override} not found for package ${packageName}`);
@@ -101,12 +103,14 @@ export async function fetchLatestPackageRelease(
  * This ensures we don't accidentally pick a package release (e.g. @pipelab/shared) as the "latest".
  */
 export async function fetchLatestDesktopRelease(
-  options: FetchReleaseOptions = {}
+  options: FetchReleaseOptions = {},
 ): Promise<GitHubRelease | null> {
   const latest = await fetchLatestPackageRelease("@pipelab/app", options);
-  
+
   if (latest) {
-    console.log(`[GitHub] Found latest desktop release: ${latest.tag_name} (${latest.prerelease ? "pre-release" : "stable"})`);
+    console.log(
+      `[GitHub] Found latest desktop release: ${latest.tag_name} (${latest.prerelease ? "pre-release" : "stable"})`,
+    );
   } else {
     console.warn(`[GitHub] No desktop releases found`);
   }

package/src/utils/remote.ts

@@ -94,7 +94,9 @@ export async function fetchPackage(
     const foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
 
     if (!foundVersion) {
-      throw new Error(`Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`);
+      throw new Error(
+        `Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`,
+      );
     }
     resolvedVersion = foundVersion;
     console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm`);
@@ -150,7 +152,13 @@ export async function runPnpm(
   },
 ) {
   const {
-    args = ["install", "--prod", "--no-lockfile", "--prefer-offline", "--no-verify-store-integrity"],
+    args = [
+      "install",
+      "--prod",
+      "--no-lockfile",
+      "--prefer-offline",
+      "--no-verify-store-integrity",
+    ],
     extraEnv = {},
     signal,
     context: ctx,
@@ -183,10 +191,7 @@ export async function runPnpm(
 /**
  * Installs a specific version of Node.js if not already present.
  */
-export async function ensureNodeJS(
-  context: PipelabContext,
-  version = DEFAULT_NODE_VERSION,
-) {
+export async function ensureNodeJS(context: PipelabContext, version = DEFAULT_NODE_VERSION) {
   const isWindows = process.platform === "win32";
   const nodeDir = context.getThirdPartyPath("node", version);
   const finalNodePath = join(nodeDir, isWindows ? "node.exe" : "bin/node");
@@ -234,7 +239,7 @@ export async function ensureNodeJS(
     await cp(sourceDir, nodeDir, { recursive: true });
     await rm(tempDir, { recursive: true, force: true });
 
-    if (!isWindows) await chmod(finalNodePath, 0o755).catch(() => { });
+    if (!isWindows) await chmod(finalNodePath, 0o755).catch(() => {});
     return finalNodePath;
   });
 }
@@ -242,10 +247,7 @@ export async function ensureNodeJS(
 /**
  * Installs the PNPM package from npm if not already present.
  */
-export async function ensurePNPM(
-  context: PipelabContext,
-  version = DEFAULT_PNPM_VERSION,
-) {
+export async function ensurePNPM(context: PipelabContext, version = DEFAULT_PNPM_VERSION) {
   const pnpmDir = context.getPackagesPath("pnpm", version);
   const pnpmPath = join(pnpmDir, "bin", "pnpm.cjs");
 
@@ -271,7 +273,12 @@ async function installDependencies(packageDir: string, packageName: string, opti
     try {
       const files = await readdir(nodeModulesPath);
       if (files.length === 0) {
-        console.warn(`[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`);
+        console.warn(
+          `[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`,
+        );
+      } else {
+        console.log(`[Fetcher] ${packageName}: Dependencies already installed, skipping.`);
+        return;
       }
     } catch (e) {
       // Continue to install if readdir fails
@@ -316,17 +323,14 @@ export async function fetchPipelabPlugin(
   options: FetchOptions,
 ): Promise<{ packageDir: string; entryPoint: string; isLocal: boolean }> {
   const { packageDir, isLocal, entryPoint } = await fetchPackage(pluginName, versionOrRange, {
-    installDeps: true,
+    installDeps: false,
     ...options,
   });
 
   // Default entry point if not provided by fetchPackage
   let finalEntryPoint = entryPoint;
   if (!finalEntryPoint) {
-    const patterns = [
-      join(packageDir, "dist", "index.mjs"),
-      join(packageDir, "index.mjs"),
-    ];
+    const patterns = [join(packageDir, "dist", "index.mjs"), join(packageDir, "index.mjs")];
     finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
   }
 
@@ -346,10 +350,7 @@ export async function fetchPipelabCli(
   // Default entry point for CLI if not provided
   let finalEntryPoint = entryPoint;
   if (!finalEntryPoint) {
-    const patterns = [
-      join(packageDir, "dist", "index.mjs"),
-      join(packageDir, "index.mjs"),
-    ];
+    const patterns = [join(packageDir, "dist", "index.mjs"), join(packageDir, "index.mjs")];
     finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
   }
 
@@ -387,7 +388,7 @@ async function tryResolveMonorepoPackage(
       typeof pkg.bin === "string"
         ? pkg.bin
         : pkg.bin?.[packageName.replace("@pipelab/", "")] ||
-        (pkg.bin ? pkg.bin[Object.keys(pkg.bin)[0]] : undefined);
+          (pkg.bin ? pkg.bin[Object.keys(pkg.bin)[0]] : undefined);
 
     // In dev, we prefer the "main" field if it points to TS, or a hardcoded src/index.ts
     const tsSource = join(packageDir, "src", "index.ts");
@@ -434,11 +435,11 @@ async function crawlMonorepoPackages(): Promise<Record<string, string>> {
               if (pkg.name) {
                 cache[pkg.name] = join(fullDir, entry.name);
               }
-            } catch (e) { }
+            } catch (e) {}
           }
         }
       }
-    } catch (e) { }
+    } catch (e) {}
   }
   return cache;
 }

package/src/utils.ts

@@ -256,7 +256,7 @@ export const executeGraphWithHistory = async ({
       // Don't await, let it run in the background
       buildHistoryStorage.applyRetentionPolicy();
     }
-    
+
     if (shouldCleanup) {
       try {
         await rm(sandboxPath, { recursive: true, force: true });