@pipelab/core-node 1.0.0-beta.11 → 1.0.0-beta.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipelab/core-node",
-  "version": "1.0.0-beta.11",
+  "version": "1.0.0-beta.13",
   "private": false,
   "description": "The Pipelab automation engine for Node.js",
   "license": "FSL-1.1-MIT",
@@ -40,8 +40,8 @@
     "type-fest": "4.39.0",
     "ws": "8.18.3",
     "yauzl": "2.10.0",
-    "@pipelab/constants": "1.0.0-beta.9",
-    "@pipelab/shared": "1.0.0-beta.7"
+    "@pipelab/constants": "1.0.0-beta.11",
+    "@pipelab/shared": "1.0.0-beta.9"
   },
   "devDependencies": {
     "@types/adm-zip": "0.5.7",
@@ -54,7 +54,7 @@
     "@types/yauzl": "2.10.3",
     "tsdown": "0.21.2",
     "typescript": "^5.0.0",
-    "@pipelab/tsconfig": "1.0.0-beta.7"
+    "@pipelab/tsconfig": "1.0.0-beta.9"
   },
   "scripts": {
     "build": "tsdown",

package/src/config.ts

@@ -3,14 +3,14 @@ import path from "node:path";
 import { ensure } from "./utils/fs-extras";
 import fs from "node:fs/promises";
 import { useLogger } from "@pipelab/shared";
-import { configRegistry, Migrator } from "@pipelab/shared";
+import { configRegistry, Migrator, normalizePipelineConfig } from "@pipelab/shared";
 
 export const getMigrator = <T>(name: string) => {
   if (configRegistry[name]) {
     return configRegistry[name] as Migrator<T>;
   }
 
-  if (name.startsWith("pipeline-")) {
+  if (name.startsWith("pipeline-") || path.isAbsolute(name) || name.endsWith(".json")) {
     return configRegistry["pipeline"] as Migrator<T>;
   }
 
@@ -50,6 +50,7 @@ export const setupConfigFile = async <T>(
       const { logger } = useLogger();
       let content = undefined;
       let originalJson: any = undefined;
+      let parseFailed = false;
 
       try {
         content = await fs.readFile(filesPath, "utf8");
@@ -58,10 +59,9 @@ export const setupConfigFile = async <T>(
         }
       } catch (e) {
         logger().error(`Error reading or parsing config ${name}:`, e);
+        parseFailed = true;
       }
 
-      // console.log("content", content);
-
       let json: any = undefined;
       try {
         json = await migrator.migrate(originalJson, {
@@ -82,11 +82,21 @@ export const setupConfigFile = async <T>(
         json = migrator.defaultValue;
       }
 
+      let normalized = false;
+      const isPipeline =
+        name.startsWith("pipeline-") ||
+        path.isAbsolute(name) ||
+        name.endsWith(".json") ||
+        name === "pipeline";
+      if (isPipeline) {
+        normalized = normalizePipelineConfig(json);
+      }
+
       const originalVersion = originalJson?.version;
       const newVersion = json?.version;
 
-      // Save back migrated config if changed or if it's a new file
-      if (originalVersion !== newVersion || content === undefined) {
+      // Save back migrated config if changed, if normalized, if it's a new file, or if parse failed
+      if (originalVersion !== newVersion || normalized || content === undefined || parseFailed) {
         try {
           await fs.writeFile(filesPath, JSON.stringify(json));
         } catch (e) {

package/src/handler-func.ts

@@ -1,13 +1,13 @@
 import { End } from "@pipelab/shared";
 import { ensureNodeJS, ensurePNPM } from "./utils/remote";
-import { Action, ActionRunner, Condition, ConditionRunner } from "./types/runner";
+import { Action, ActionRunner } from "./types/runner";
 import { InputsDefinition } from "@pipelab/shared";
 import { usePlugins } from "@pipelab/shared";
 import { isRequired } from "@pipelab/shared";
 import { mkdir, stat } from "node:fs/promises";
 import { PipelabContext } from "./context";
 import { useLogger } from "@pipelab/shared";
-import { BlockCondition } from "@pipelab/shared";
+
 import { HandleListenerSendFn } from "./handlers";
 import path from "node:path";
 const { join } = path;
@@ -38,81 +38,6 @@ const checkParams = (definitionParams: InputsDefinition, elementParams: Record<s
   }
 };
 
-export const handleConditionExecute = async (
-  nodeId: string,
-  pluginId: string,
-  params: BlockCondition["params"],
-  cwd: string,
-  context: PipelabContext,
-): Promise<End<"condition:execute">> => {
-  const ctx = context;
-  const { plugins } = usePlugins();
-  const { logger } = useLogger();
-
-  const node = plugins.value
-    .find((plugin) => plugin.id === pluginId)
-    ?.nodes.find((node: any) => node.node.id === nodeId) as
-    | {
-        node: Condition;
-        runner: ConditionRunner<any>;
-      }
-    | undefined;
-
-  if (!node) {
-    return {
-      type: "error",
-      ipcError: "Node not found",
-    };
-  }
-
-  try {
-    const s = await stat(cwd);
-    if (!s.isDirectory()) {
-      throw new Error(`Execution directory "${cwd}" exists but is not a directory.`);
-    }
-  } catch (e: any) {
-    if (e.code === "ENOENT") {
-      await mkdir(cwd, { recursive: true });
-    } else {
-      throw e;
-    }
-  }
-
-  try {
-    checkParams(node.node.params, params);
-    const resolvedInputs = params; // await resolveConditionInputs(params, node.node, steps)
-
-    const result = await node.runner({
-      inputs: resolvedInputs,
-      log: (...args) => {
-        logger().info(`[${node.node.name}]`, ...args);
-      },
-      meta: {
-        definition: "",
-      },
-      setMeta: () => {
-        logger().info("set meta defined here");
-      },
-      cwd,
-      context: ctx,
-    });
-
-    return {
-      type: "success",
-      result: {
-        outputs: {},
-        value: result,
-      },
-    };
-  } catch (e) {
-    logger().error("Error in condition execution:", e);
-    return {
-      type: "error",
-      ipcError: String(e),
-    };
-  }
-};
-
 export const handleActionExecute = async (
   nodeId: string,
   pluginId: string,

package/src/handlers/config.ts

@@ -2,6 +2,8 @@ import { useAPI } from "../ipc-core";
 import { useLogger, configRegistry } from "@pipelab/shared";
 import { setupConfigFile, getMigrator } from "../config";
 import { PipelabContext } from "../context";
+import fs from "node:fs/promises";
+import path from "node:path";
 
 export const registerConfigHandlers = (context: PipelabContext) => {
   const { handle } = useAPI();
@@ -106,4 +108,51 @@ export const registerConfigHandlers = (context: PipelabContext) => {
       });
     }
   });
+
+  handle("config:delete", async (_, { send, value }) => {
+    const { config: name } = value;
+    logger().info("config:delete", name);
+
+    try {
+      // Config could be an absolute path (for external files/pipelines)
+      // or a name/identifier (for internal config files stored in Pipelab's app data directory).
+      const isAbsolutePath = path.isAbsolute(name);
+      const filesPath = isAbsolutePath ? name : context.getConfigPath(`${name}.json`);
+
+      await fs.rm(filesPath, { force: true });
+
+      // Clean up versioned backups too
+      const parsedPath = path.parse(filesPath);
+      const dirEntries = await fs.readdir(parsedPath.dir).catch(() => [] as string[]);
+      const prefix = `${parsedPath.name}.v`;
+      const suffix = `.json`;
+      for (const entry of dirEntries) {
+        if (entry.startsWith(prefix) && entry.endsWith(suffix)) {
+          const backupPath = path.join(parsedPath.dir, entry);
+          await fs.rm(backupPath, { force: true }).catch((err) => {
+            logger().error(`Failed to delete backup ${backupPath}:`, err);
+          });
+        }
+      }
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: {
+            result: "ok",
+          },
+        },
+      });
+    } catch (e) {
+      logger().error(`config:delete error for ${name}:`, e);
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e instanceof Error ? e.message : `Unable to delete config ${name}`,
+        },
+      });
+    }
+  });
 };

package/src/handlers/engine.ts

@@ -3,7 +3,7 @@ import { PipelabContext } from "../context";
 import { useLogger } from "@pipelab/shared";
 import { getFinalPlugins, executeGraphWithHistory } from "../utils";
 import { presets } from "../presets/list";
-import { handleActionExecute, handleConditionExecute } from "../handler-func";
+import { handleActionExecute } from "../handler-func";
 import { generateTempFolder } from "../utils/fs-extras";
 import { tmpdir } from "node:os";
 import { setupConfigFile } from "../config";
@@ -37,12 +37,6 @@ export const registerEngineHandlers = (context: PipelabContext) => {
     });
   });
 
-  handle("condition:execute", async (_, { value }) => {
-    const { nodeId, params, pluginId } = value;
-    const cwd = await generateTempFolder(tmpdir());
-    await handleConditionExecute(nodeId, pluginId, params, cwd, context);
-  });
-
   let abortControllerGraph: undefined | AbortController = undefined;
 
   const effectiveActionExecute = async (

package/src/handlers/index.ts

@@ -6,6 +6,7 @@ import { registerEngineHandlers } from "./engine";
 import { registerAgentsHandlers } from "./agents";
 import { registerAuthHandlers } from "./auth";
 import { registerSystemHandlers } from "./system";
+import { registerPluginsHandlers } from "./plugins";
 import { builtInPlugins } from "../plugins-registry";
 import { usePlugins } from "@pipelab/shared";
 import { PipelabContext } from "../context";
@@ -24,6 +25,7 @@ export const registerAllHandlers = async (options: {
   registerAgentsHandlers(context);
   registerAuthHandlers(context);
   registerSystemHandlers(options);
+  registerPluginsHandlers(context);
 
   const { registerPlugins } = usePlugins();
   // Execute in the background! The plugins will be dynamically registered and broadcasted to the UI.

package/src/handlers/plugins.ts

@@ -0,0 +1,310 @@
+import { useAPI } from "../ipc-core";
+import { PipelabContext, isDev, projectRoot } from "../context";
+import pacote from "pacote";
+import { rm } from "node:fs/promises";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { usePlugins } from "@pipelab/shared";
+import { webSocketServer } from "../websocket-server";
+import { fetchPipelabPlugin } from "../utils/remote";
+import { loadCustomPlugin, findInstalledPlugins } from "../plugins-registry";
+
+// Maps workspace plugin package names (e.g. "@pipelab/plugin-steam") to their physical directory paths.
+// This is populated at startup in dev mode, supporting plugin folders whose directory names
+// differ from their actual package.json name.
+const localPluginsMap = new Map<string, string>();
+
+if (isDev && projectRoot) {
+  const pluginsDir = join(projectRoot, "plugins");
+  if (existsSync(pluginsDir)) {
+    try {
+      const entries = readdirSync(pluginsDir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.isDirectory()) {
+          const pkgPath = join(pluginsDir, entry.name, "package.json");
+          if (existsSync(pkgPath)) {
+            try {
+              const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+              if (pkg.name) {
+                localPluginsMap.set(pkg.name, join(pluginsDir, entry.name));
+              }
+            } catch (e) {
+              console.error(`[Plugins] Failed to parse package.json for ${entry.name}:`, e);
+            }
+          }
+        }
+      }
+    } catch (e) {
+      console.error(`[Plugins] Failed to scan local workspace plugins directory:`, e);
+    }
+  }
+}
+
+/**
+ * Resolves a requested plugin version.
+ * If the plugin is present in the local workspace map during development,
+ * its version is overridden to "local" so Pipelab loads it from the source files.
+ */
+export function resolvePluginVersion(packageName: string, requestedVersion?: string): string {
+  if (
+    isDev &&
+    projectRoot &&
+    process.env.PIPELAB_FORCE_NPM !== "true" &&
+    localPluginsMap.has(packageName)
+  ) {
+    return "local";
+  }
+  return requestedVersion || "latest";
+}
+
+export const registerPluginsHandlers = (context: PipelabContext) => {
+  const { handle } = useAPI();
+
+  handle("plugin:search", async (_, { send, value }) => {
+    try {
+      const query = value.query || "";
+      const text = query ? `${query} keywords:pipelab-plugin` : "keywords:pipelab-plugin";
+      const url = `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(text)}&size=50`;
+
+      const response = await fetch(url);
+      if (!response.ok) {
+        throw new Error(`NPM registry search failed with status ${response.status}`);
+      }
+
+      const data = await response.json();
+      const results = (data.objects || []).map((obj: any) => ({
+        name: obj.package.name,
+        version: obj.package.version,
+        description: obj.package.description,
+        keywords: obj.package.keywords,
+        date: obj.package.date,
+      }));
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: { results },
+        },
+      });
+    } catch (e: any) {
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e.message || "Failed to search npm registry",
+        },
+      });
+    }
+  });
+
+  handle("plugin:get-details", async (_, { send, value }) => {
+    try {
+      const { packageName } = value;
+      const packument = await pacote.packument(packageName, { fullMetadata: true });
+      const latestVersion =
+        packument["dist-tags"]?.latest || Object.keys(packument.versions).pop() || "0.0.0";
+      const latestPkg = packument.versions[latestVersion];
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: {
+            name: packument.name,
+            latestVersion,
+            versions: Object.keys(packument.versions).reverse(),
+            description: latestPkg?.description,
+          },
+        },
+      });
+    } catch (e: any) {
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e.message || "Failed to get plugin details",
+        },
+      });
+    }
+  });
+
+  handle("plugin:install", async (_, { send, value }) => {
+    try {
+      const { packageName, version } = value;
+      const mappedVersion = resolvePluginVersion(packageName, version);
+      console.log(`[Plugins] Installing ${packageName}@${mappedVersion}...`);
+
+      const { packageDir } = await fetchPipelabPlugin(packageName, mappedVersion, {
+        context,
+        installDeps: false,
+      });
+
+      const plugin = await loadCustomPlugin(packageName, mappedVersion, { context });
+      if (!plugin) {
+        throw new Error("Failed to load installed plugin module.");
+      }
+
+      const { registerPlugins } = usePlugins();
+      registerPlugins([plugin]);
+
+      webSocketServer.broadcast("plugin:loaded", { plugin });
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: { result: "ok" },
+        },
+      });
+    } catch (e: any) {
+      console.error(`[Plugins] Installation failed for ${value.packageName}:`, e);
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e.message || "Failed to install plugin",
+        },
+      });
+    }
+  });
+
+  handle("plugin:uninstall", async (_, { send, value }) => {
+    try {
+      const { packageName } = value;
+      console.log(`[Plugins] Uninstalling plugin ${packageName}...`);
+      const targetDir = context.getPackagesPath(packageName);
+
+      if (existsSync(targetDir)) {
+        await rm(targetDir, { recursive: true, force: true });
+      }
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: { result: "ok" },
+        },
+      });
+    } catch (e: any) {
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e.message || "Failed to uninstall plugin",
+        },
+      });
+    }
+  });
+
+  handle("plugin:list-installed", async (_, { send }) => {
+    try {
+      const packagesDir = context.getPackagesPath();
+      const rawInstalled = await findInstalledPlugins(packagesDir);
+
+      const DEFAULT_PLUGIN_IDS = [
+        "construct",
+        "filesystem",
+        "system",
+        "steam",
+        "itch",
+        "electron",
+        "discord",
+        "poki",
+        "nvpatch",
+        "tauri",
+        "minify",
+        "netlify",
+      ];
+
+      const installed = rawInstalled
+        .filter((item) => {
+          const isDefault = DEFAULT_PLUGIN_IDS.some((id) => item.name === `@pipelab/plugin-${id}`);
+          return !isDefault;
+        })
+        .map((item) => ({
+          name: item.name,
+          version: item.version,
+          description: item.description,
+        }));
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: { installed },
+        },
+      });
+    } catch (e: any) {
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e.message || "Failed to list installed plugins",
+        },
+      });
+    }
+  });
+
+  // Ensures all required plugin IDs are loaded, JIT-installing any that are missing.
+  // Called before opening a pipeline in the editor.
+  handle("plugin:ensure-loaded", async (_, { send, value }) => {
+    const { pluginIds, plugins } = value;
+    const { plugins: registeredPlugins, registerPlugins } = usePlugins();
+    const loaded: string[] = [];
+    const failed: string[] = [];
+
+    const pluginsToEnsure = new Set<string>();
+    if (Array.isArray(pluginIds)) {
+      for (const id of pluginIds) {
+        if (id) pluginsToEnsure.add(id);
+      }
+    }
+    if (plugins && typeof plugins === "object") {
+      for (const id of Object.keys(plugins)) {
+        if (id) pluginsToEnsure.add(id);
+      }
+    }
+
+    for (const packageName of pluginsToEnsure) {
+      const mappedVersion = resolvePluginVersion(packageName);
+
+      const isRegistered = registeredPlugins.value.some((p) => {
+        if (p.packageName !== packageName) return false;
+        if (mappedVersion === "latest") return true;
+        return p.version === mappedVersion;
+      });
+
+      if (isRegistered) {
+        continue; // already loaded, nothing to do
+      }
+
+      try {
+        console.log(
+          `[Plugins] JIT loading plugin "${packageName}@${mappedVersion}" for pipeline...`,
+        );
+        const plugin = await loadCustomPlugin(packageName, mappedVersion, { context });
+        if (plugin) {
+          registerPlugins([plugin]);
+          webSocketServer.broadcast("plugin:loaded", { plugin });
+          loaded.push(packageName);
+          console.log(`[Plugins] JIT loaded "${packageName}" successfully.`);
+        } else {
+          console.warn(`[Plugins] JIT load for "${packageName}" returned no plugin.`);
+          failed.push(packageName);
+        }
+      } catch (e: any) {
+        console.error(`[Plugins] JIT load failed for "${packageName}":`, e);
+        failed.push(packageName);
+      }
+    }
+
+    send({
+      type: "end",
+      data: {
+        type: "success",
+        result: { loaded, failed },
+      },
+    });
+  });
+};