@pipelab/core-node 1.0.0-beta.2 → 1.0.0-beta.21

package/src/plugins-registry.ts

@@ -1,32 +1,84 @@
 import { ensureNodeJS, ensurePNPM, fetchPipelabPlugin } from "./utils/remote";
 import { pathToFileURL } from "node:url";
-import { readdir } from "node:fs/promises";
+import { readdir, readFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
-import { PipelabContext } from "./context";
+import { join } from "node:path";
+import { PipelabContext, isDev, projectRoot } from "./context";
 import { sendStartupProgress } from "./server";
 
-const DEFAULT_PLUGIN_IDS = [
-  "construct",
-  "filesystem",
-  "system",
-  "steam",
-  "itch",
-  "electron",
-  "discord",
-  "poki",
-  "nvpatch",
-  "tauri",
-  "minify",
-  "netlify",
-];
+const enhancePluginDefinition = async (
+  plugin: any,
+  packageDir: string,
+  fallbackName: string,
+  fallbackVersion: string,
+) => {
+  if (!plugin) return plugin;
+
+  let packageName = fallbackName;
+  let version = fallbackVersion;
+  let pipelabMeta: any = null;
+  let pkgDescription = "";
+
+  try {
+    const pkgJsonPath = join(packageDir, "package.json");
+    if (existsSync(pkgJsonPath)) {
+      const pkgContent = await readFile(pkgJsonPath, "utf8");
+      const pkg = JSON.parse(pkgContent);
+      if (pkg.name) packageName = pkg.name;
+      if (pkg.version) version = pkg.version;
+      if (pkg.description) pkgDescription = pkg.description;
+      if (pkg.pipelab) pipelabMeta = pkg.pipelab;
+    }
+  } catch (e) {
+    console.error(`[Plugins] Failed to read package.json in ${packageDir}:`, e);
+  }
+
+  plugin.packageName = packageName;
+  plugin.id = packageName;
+  plugin.version = fallbackVersion === "local" ? "local" : version;
+  plugin.isOfficial = packageName.startsWith("@pipelab/");
+
+  // Dynamically resolve name, description, and icon from package.json
+  plugin.name = pipelabMeta?.name || packageName;
+  plugin.description = pipelabMeta?.description || pkgDescription || "";
+
+  if (pipelabMeta?.icon) {
+    if (typeof pipelabMeta.icon === "string") {
+      let iconPath = pipelabMeta.icon;
+      if (iconPath.startsWith(".")) {
+        iconPath = pathToFileURL(join(packageDir, iconPath)).href;
+      }
+      plugin.icon = {
+        type: "image",
+        image: iconPath,
+      };
+    } else {
+      plugin.icon = pipelabMeta.icon;
+    }
+  } else {
+    plugin.icon = {
+      type: "icon",
+      icon: "pi pi-box",
+    };
+  }
+
+  return plugin;
+};
 
 export const loadPipelabPlugin = async (id: string, options: { context: PipelabContext }) => {
+  const start = Date.now();
   try {
-    const packageName = `@pipelab/plugin-${id}`;
-    const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, "latest", {
-      context: options.context,
-      installDeps: false,
-    });
+    const packageName = id;
+    const fetchStart = Date.now();
+    const { packageDir, entryPoint } = await fetchPipelabPlugin(
+      packageName,
+      options.context.releaseTag,
+      {
+        context: options.context,
+        installDeps: false,
+      },
+    );
+    const fetchDuration = Date.now() - fetchStart;
 
     console.log(`[Plugins] [${id}] Attempting to import from: ${entryPoint}`);
     if (!existsSync(entryPoint)) {
@@ -37,11 +89,19 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
       } catch (e) {}
     }
 
+    const importStart = Date.now();
     const pluginModule = await import(pathToFileURL(entryPoint).href);
-    console.log(`[Plugins] [${id}] Successfully loaded from: ${packageDir}`);
-    return pluginModule.default;
+    const importDuration = Date.now() - importStart;
+    const totalDuration = Date.now() - start;
+    console.log(
+      `[Plugins] [${id}] Successfully loaded from: ${packageDir} (fetch: ${fetchDuration}ms, import: ${importDuration}ms, total: ${totalDuration}ms)`,
+    );
+
+    const plugin = pluginModule.default;
+    await enhancePluginDefinition(plugin, packageDir, packageName, options.context.releaseTag);
+    return plugin;
   } catch (e: any) {
-    console.error(`[Plugins] [${id}] CRITICAL: Failed to load:`, e);
+    console.error(`[Plugins] [${id}] CRITICAL: Failed to load after ${Date.now() - start}ms:`, e);
     if (e.code === "ERR_MODULE_NOT_FOUND") {
       console.error(
         `[Plugins] [${id}] This usually means a dependency is missing in the plugin's node_modules.`,
@@ -51,34 +111,216 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
   }
 };
 
-export const builtInPlugins = async (options: { context: PipelabContext }) => {
+export const loadCustomPlugin = async (
+  packageName: string,
+  version: string,
+  options: { context: PipelabContext },
+) => {
+  const start = Date.now();
+  try {
+    const fetchStart = Date.now();
+    const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, version, {
+      context: options.context,
+      installDeps: false,
+    });
+    const fetchDuration = Date.now() - fetchStart;
+
+    console.log(
+      `[Plugins] [${packageName}] Attempting to import custom plugin from: ${entryPoint}`,
+    );
+    if (!existsSync(entryPoint)) {
+      console.error(
+        `[Plugins] [${packageName}] CRITICAL: Plugin entry point not found at ${entryPoint}`,
+      );
+      return null;
+    }
+
+    const importStart = Date.now();
+    const pluginModule = await import(pathToFileURL(entryPoint).href);
+    const importDuration = Date.now() - importStart;
+    const totalDuration = Date.now() - start;
+    console.log(
+      `[Plugins] [${packageName}] Successfully loaded custom plugin from: ${packageDir} (fetch: ${fetchDuration}ms, import: ${importDuration}ms, total: ${totalDuration}ms)`,
+    );
+
+    const plugin = pluginModule.default;
+    await enhancePluginDefinition(plugin, packageDir, packageName, version);
+    return plugin;
+  } catch (e: any) {
+    console.error(
+      `[Plugins] [${packageName}] CRITICAL: Failed to load after ${Date.now() - start}ms:`,
+      e,
+    );
+    return null;
+  }
+};
+
+export async function findInstalledPlugins(
+  packagesDir: string,
+): Promise<Array<{ name: string; version: string; packageDir: string; description: string }>> {
+  const installed: Array<{
+    name: string;
+    version: string;
+    packageDir: string;
+    description: string;
+  }> = [];
+  if (!existsSync(packagesDir)) return installed;
+
+  async function scan(dir: string, depth = 0) {
+    if (depth > 4) return;
+    try {
+      const entries = await readdir(dir, { withFileTypes: true });
+      const hasPkgJson = entries.some((e) => e.isFile() && e.name === "package.json");
+      if (hasPkgJson) {
+        try {
+          const content = await readFile(join(dir, "package.json"), "utf8");
+          const pkg = JSON.parse(content);
+          if (pkg.name && pkg.name !== "pnpm") {
+            installed.push({
+              name: pkg.name,
+              version: pkg.version || "0.0.0",
+              packageDir: dir,
+              description: pkg.description || "",
+            });
+          }
+        } catch (e) {
+          // ignore invalid package.json
+        }
+        return;
+      }
+
+      for (const entry of entries) {
+        if (entry.isDirectory() && entry.name !== "node_modules" && !entry.name.startsWith(".")) {
+          await scan(join(dir, entry.name), depth + 1);
+        }
+      }
+    } catch (e) {
+      // ignore errors
+    }
+  }
+
+  await scan(packagesDir);
+  return installed;
+}
+
+export const builtInPlugins = async (options: { context: PipelabContext }): Promise<void> => {
   console.log("[Plugins] Starting background plugin loading...");
 
   // Pre-ensure Node.js and PNPM once in parallel so plugins don't have to wait for them
   sendStartupProgress("Preparing environment...");
+  const envStart = Date.now();
   await Promise.all([ensureNodeJS(options.context), ensurePNPM(options.context)]);
+  console.log(`[Plugins] Environment preparation took ${Date.now() - envStart}ms`);
 
   const { usePlugins } = await import("@pipelab/shared");
   const { registerPlugins } = usePlugins();
   const { webSocketServer } = await import("./index");
 
+  // Broadcast ready signal immediately so the UI launches while plugins load in the background
+  webSocketServer.broadcast("startup:progress", { type: "ready" });
+
   // Load plugins asynchronously in the background
-  Promise.allSettled(
-    DEFAULT_PLUGIN_IDS.map(async (id) => {
-      sendStartupProgress(`Loading plugin: ${id}`);
-      const plugin = await loadPipelabPlugin(id, options);
-      if (plugin) {
-        registerPlugins([plugin]);
-        webSocketServer.broadcast("plugin:loaded", { plugin });
+  (async () => {
+    const totalStart = Date.now();
+    const pluginsToLoad = new Map<string, string>(); // packageName -> version
+
+    // Load default/native plugins by default
+    const defaultPlugins = [
+      "@pipelab/plugin-construct",
+      "@pipelab/plugin-filesystem",
+      "@pipelab/plugin-system",
+      "@pipelab/plugin-steam",
+      "@pipelab/plugin-itch",
+      "@pipelab/plugin-electron",
+      "@pipelab/plugin-discord",
+      "@pipelab/plugin-poki",
+      "@pipelab/plugin-nvpatch",
+      "@pipelab/plugin-tauri",
+      "@pipelab/plugin-minify",
+      "@pipelab/plugin-netlify",
+    ];
+    for (const name of defaultPlugins) {
+      pluginsToLoad.set(name, "latest");
+    }
+
+    if (isDev && projectRoot) {
+      const pluginsDir = join(projectRoot, "plugins");
+      if (existsSync(pluginsDir)) {
+        try {
+          const entries = await readdir(pluginsDir, { withFileTypes: true });
+          for (const entry of entries) {
+            if (entry.isDirectory()) {
+              const pkgPath = join(pluginsDir, entry.name, "package.json");
+              if (existsSync(pkgPath)) {
+                try {
+                  const pkgContent = await readFile(pkgPath, "utf-8");
+                  const pkg = JSON.parse(pkgContent);
+                  if (
+                    pkg.name &&
+                    pkg.name.startsWith("@pipelab/plugin-") &&
+                    pkg.name !== "@pipelab/plugin-core"
+                  ) {
+                    pluginsToLoad.set(pkg.name, "local");
+                  }
+                } catch (e) {
+                  console.error(`[Plugins] Failed to parse package.json for ${entry.name}:`, e);
+                }
+              }
+            }
+          }
+        } catch (e) {
+          console.error(`[Plugins] Failed to scan local workspace plugins directory:`, e);
+        }
+      }
+    }
+
+    try {
+      const { setupSettingsConfigFile } = await import("./config");
+      const settingsManager = await setupSettingsConfigFile(options.context);
+      const settingsConfig = await settingsManager.getConfig();
+      const settingsPlugins = settingsConfig?.plugins || [];
+
+      for (const plugin of settingsPlugins) {
+        if (plugin.name) {
+          if (plugin.enabled) {
+            if (!pluginsToLoad.has(plugin.name)) {
+              pluginsToLoad.set(plugin.name, "latest");
+            }
+          } else {
+            // If explicitly disabled in settings, remove it from the loading list
+            pluginsToLoad.delete(plugin.name);
+          }
+        }
+      }
+    } catch (e) {
+      console.error(`[Plugins] Failed to load settings config on startup:`, e);
+    }
+
+    console.log(`[Plugins] Total plugins to load on startup:`, Array.from(pluginsToLoad.entries()));
+
+    // Now load all collected plugins in parallel
+    const loadPromises = Array.from(pluginsToLoad.entries()).map(async ([packageName, version]) => {
+      sendStartupProgress(`Loading plugin: ${packageName}`);
+      const pluginStart = Date.now();
+      try {
+        const plugin = await loadCustomPlugin(packageName, version, options);
+        if (plugin) {
+          registerPlugins([plugin]);
+          webSocketServer.broadcast("plugin:loaded", { plugin });
+          console.log(
+            `[Plugins] Loaded ${packageName}@${version} in ${Date.now() - pluginStart}ms`,
+          );
+        }
+      } catch (err) {
+        console.error(`[Plugins] Failed to load plugin ${packageName} at startup:`, err);
       }
-    }),
-  ).then(() => {
-    console.log("[Plugins] All default plugins loaded.");
+    });
+    await Promise.all(loadPromises);
+
+    console.log(`[Plugins] All startup plugins loaded in ${Date.now() - totalStart}ms.`);
     sendStartupProgress("All plugins loaded.");
     setTimeout(() => {
       webSocketServer.broadcast("startup:progress", { type: "done" });
     }, 2000);
-  });
-
-  return [];
+  })();
 };

package/src/presets/c3toSteam.ts

@@ -4,7 +4,7 @@ export const c3toSteamPreset: PresetFn = async () => {
   const startId = "manual-start";
 
   const data: SavedFile = {
-    version: "3.0.0",
+    version: "5.0.0",
     name: "Construct 3 to Steam",
     description: "A basic project to get you started with Construct 3 and Steam",
     variables: [],
@@ -13,8 +13,9 @@ export const c3toSteamPreset: PresetFn = async () => {
         {
           type: "event",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "manual",
+            version: "latest",
           },
           uid: startId,
           params: {},
@@ -26,7 +27,8 @@ export const c3toSteamPreset: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "export-construct-project",
-            pluginId: "construct",
+            pluginId: "@pipelab/plugin-construct",
+            version: "latest",
           },
           params: {
             file: {
@@ -64,7 +66,8 @@ export const c3toSteamPreset: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "unzip-file-node",
-            pluginId: "filesystem",
+            pluginId: "@pipelab/plugin-filesystem",
+            version: "latest",
           },
           params: {
             file: {
@@ -78,7 +81,8 @@ export const c3toSteamPreset: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "electron:package:v2",
-            pluginId: "electron",
+            pluginId: "@pipelab/plugin-electron",
+            version: "latest",
           },
           params: {
             arch: {
@@ -208,7 +212,8 @@ export const c3toSteamPreset: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "steam-upload",
-            pluginId: "steam",
+            pluginId: "@pipelab/plugin-steam",
+            version: "latest",
           },
           params: {
             sdk: {
@@ -251,7 +256,8 @@ export const c3toSteamPreset: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "fs:open-in-explorer",
-            pluginId: "filesystem",
+            pluginId: "@pipelab/plugin-filesystem",
+            version: "latest",
           },
           params: {
             path: {

package/src/presets/demo.ts

@@ -9,7 +9,7 @@ export const demoPreset: PresetFn = async () => {
   const logOkId = "log-ok";
 
   const data: SavedFile = {
-    version: "1.0.0",
+    version: "5.0.0",
     variables: [],
     name: "demo",
     description: "demo",
@@ -18,7 +18,7 @@ export const demoPreset: PresetFn = async () => {
         {
           type: "event",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "manual",
           },
           uid: startId,
@@ -27,7 +27,7 @@ export const demoPreset: PresetFn = async () => {
         {
           type: "action",
           origin: {
-            pluginId: "construct",
+            pluginId: "@pipelab/plugin-construct",
             nodeId: "export-construct-project",
           },
           uid: exportConstructProjectId,
@@ -41,7 +41,7 @@ export const demoPreset: PresetFn = async () => {
         {
           type: "action",
           origin: {
-            pluginId: "filesystem",
+            pluginId: "@pipelab/plugin-filesystem",
             nodeId: "list-files-node",
           },
           uid: listFilesNodeId,
@@ -53,7 +53,7 @@ export const demoPreset: PresetFn = async () => {
         {
           type: "loop",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "for",
           },
 
@@ -64,7 +64,7 @@ export const demoPreset: PresetFn = async () => {
             {
               type: "condition",
               origin: {
-                pluginId: "filesystem",
+                pluginId: "@pipelab/plugin-filesystem",
                 nodeId: "is-file",
               },
               uid: "is-file-condition",
@@ -76,7 +76,7 @@ export const demoPreset: PresetFn = async () => {
                 {
                   type: "action",
                   origin: {
-                    pluginId: "system",
+                    pluginId: "@pipelab/plugin-system",
                     nodeId: "log",
                   },
                   params: {
@@ -89,7 +89,7 @@ export const demoPreset: PresetFn = async () => {
                 {
                   type: "action",
                   origin: {
-                    pluginId: "system",
+                    pluginId: "@pipelab/plugin-system",
                     nodeId: "log",
                   },
                   params: {
@@ -105,7 +105,7 @@ export const demoPreset: PresetFn = async () => {
         {
           type: "action",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "log",
           },
           uid: logOkId,

package/src/presets/list.ts

@@ -1,6 +1,4 @@
 // import { demoPreset } from './demo'
-// import { ifPreset } from './if'
-// import { loopPreset } from './loop'
 // import { testC3Unzip } from './test-c3-unzip'
 // import { testC3Offline } from './test-c3-offline'
 import { c3toSteamPreset } from "./c3toSteam";
@@ -13,8 +11,6 @@ export const presets = async () => {
   const moreToComeVal = await moreToCome();
 
   // const demoPresetVal = await demoPreset()
-  // const ifPresetVal = await ifPreset()
-  // const loopPresetVal = await loopPreset()
   // const testC3UnzipVal = await testC3Unzip()
   // const testC3OfflineVal = await testC3Offline()
   return {
@@ -22,8 +18,6 @@ export const presets = async () => {
     c3toSteam: c3toSteamVal,
     moreToCome: moreToComeVal,
     // demo: demoPresetVal,
-    // if: ifPresetVal,
-    // loop: loopPresetVal,
     // testC3Unzip: testC3UnzipVal,
     // testC3Offline: testC3OfflineVal,
   };

package/src/presets/moreToCome.ts

@@ -4,7 +4,7 @@ export const moreToCome: PresetFn = async () => {
   const startId = "manual-start";
 
   const data: SavedFile = {
-    version: "3.0.0",
+    version: "5.0.0",
     name: "More to come!",
     description: "Do not hesitate to suggest templates you would see here",
     variables: [],
@@ -13,8 +13,9 @@ export const moreToCome: PresetFn = async () => {
         {
           type: "event",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "manual",
+            version: "latest",
           },
           uid: startId,
           params: {},

package/src/presets/newProject.ts

@@ -4,7 +4,7 @@ export const newProjectPreset: PresetFn = async () => {
   const startId = "manual-start";
 
   const data: SavedFile = {
-    version: "3.0.0",
+    version: "5.0.0",
     name: "Empty project",
     description: "A default project with no tasks added",
     variables: [],
@@ -13,8 +13,9 @@ export const newProjectPreset: PresetFn = async () => {
         {
           type: "event",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "manual",
+            version: "latest",
           },
           uid: startId,
           params: {},

package/src/presets/test-c3-offline.ts

@@ -5,7 +5,7 @@ export const testC3Offline: PresetFn = async () => {
   const steamUpload = "steam-upload-node";
 
   const data: SavedFile = {
-    version: "3.0.0",
+    version: "5.0.0",
     variables: [],
     name: "C3 test without export",
     description: "C3 test without export",
@@ -14,8 +14,9 @@ export const testC3Offline: PresetFn = async () => {
         {
           type: "event",
           origin: {
-            pluginId: "system",
+            pluginId: "@pipelab/plugin-system",
             nodeId: "manual",
+            version: "latest",
           },
           uid: "manual-start",
           params: {},
@@ -27,15 +28,22 @@ export const testC3Offline: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "package-to-electron",
-            pluginId: "electron",
+            pluginId: "@pipelab/plugin-electron",
+            version: "latest",
           },
           params: {
             "input-folder": {
               editor: "editor",
               value: `/home/quentin/Documents/Cyn Assets/app`,
             },
-            arch: undefined,
-            platform: undefined,
+            arch: {
+              editor: "simple",
+              value: undefined,
+            },
+            platform: {
+              editor: "simple",
+              value: undefined,
+            },
           },
         },
         {
@@ -43,7 +51,8 @@ export const testC3Offline: PresetFn = async () => {
           type: "action",
           origin: {
             nodeId: "steam-upload",
-            pluginId: "steam",
+            pluginId: "@pipelab/plugin-steam",
+            version: "latest",
           },
           params: {
             folder: {