@pipelab/core-node 1.0.1-beta.23

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@pipelab/core-node",
+  "version": "1.0.1-beta.23",
+  "private": false,
+  "description": "The Pipelab automation engine for Node.js",
+  "license": "FSL-1.1-MIT",
+  "author": "CynToolkit",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/CynToolkit/pipelab.git",
+    "directory": "packages/core-node"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.mjs"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "adm-zip": "0.5.16",
+    "archiver": "7.0.1",
+    "es-toolkit": "1.25.2",
+    "execa": "9.5.2",
+    "nanoid": "5.0.7",
+    "pacote": "21.5.0",
+    "semver": "^7.7.4",
+    "slash": "3.0.0",
+    "tar": "6.2.1",
+    "type-fest": "4.39.0",
+    "ws": "8.18.3",
+    "yauzl": "2.10.0",
+    "@pipelab/constants": "1.0.1-beta.19",
+    "@pipelab/shared": "2.0.1-beta.20"
+  },
+  "devDependencies": {
+    "@types/adm-zip": "0.5.7",
+    "@types/archiver": "7.0.0",
+    "@types/node": "24.12.2",
+    "@types/pacote": "11.1.8",
+    "@types/semver": "^7.7.1",
+    "@types/tar": "7.0.87",
+    "@types/ws": "8.18.1",
+    "@types/yauzl": "2.10.3",
+    "tsdown": "0.21.2",
+    "typescript": "^5.0.0",
+    "@pipelab/tsconfig": "1.0.1-beta.19"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "format": "oxfmt .",
+    "lint": "oxlint ."
+  }
+}

package/src/api.ts

@@ -0,0 +1,115 @@
+import { nanoid } from "nanoid";
+import { klona } from "klona";
+import { toRaw } from "vue";
+
+import type { Tagged } from "type-fest";
+import { ILogObjMeta } from "tslog";
+import { useLogger } from "@pipelab/shared";
+
+import {
+  UpdateStatus,
+  RendererChannels,
+  RendererData,
+  RendererEvents,
+  RendererEnd,
+  RendererMessage,
+  RequestId,
+  HandleListenerRendererSendFn,
+  HandleListenerRenderer as BaseHandleListenerRenderer,
+} from "@pipelab/shared";
+
+export type {
+  UpdateStatus,
+  RendererChannels,
+  RendererData,
+  RendererEvents,
+  RendererEnd,
+  RendererMessage,
+  RequestId,
+  HandleListenerRendererSendFn,
+};
+
+export type HandleListenerRenderer<KEY extends RendererChannels> = (
+  event: Electron.IpcMainInvokeEvent,
+  data: { value: RendererData<KEY>; send: HandleListenerRendererSendFn<KEY> },
+) => Promise<void>;
+
+export type ListenerMain<KEY extends RendererChannels> = (
+  event: Electron.IpcMainEvent,
+  data: RendererEvents<KEY>,
+) => Promise<void>;
+
+export const usePluginAPI = (browserWindow: any) => {
+  const { logger } = useLogger();
+  /**
+   * Send an order
+   */
+  const send = <KEY extends RendererChannels>(channel: KEY, args?: RendererData<KEY>) => {
+    if (!browserWindow || browserWindow.isDestroyed()) return;
+    browserWindow.webContents.send(channel, args);
+  };
+
+  const on = <KEY extends RendererChannels>(
+    channel: KEY | string,
+    listener: (event: any, data: RendererEvents<KEY>) => void,
+  ) => {
+    if (!browserWindow || browserWindow.isDestroyed()) return () => {};
+    const ipcMain = browserWindow.webContents.ipc.on(channel, listener);
+
+    const cancel = () => {
+      if (browserWindow.isDestroyed()) return;
+      ipcMain.removeListener(channel, listener);
+    };
+
+    return cancel;
+  };
+
+  /**
+   * Send an order and wait for it's execution
+   */
+  const execute = async <KEY extends RendererChannels>(
+    channel: KEY,
+    data?: RendererData<KEY>,
+    listener?: ListenerMain<KEY>,
+  ) => {
+    const newId = nanoid() as RequestId;
+    // eslint-disable-next-line no-async-promise-executor
+    return new Promise<RendererEnd<KEY>>(async (resolve, reject) => {
+      const message: RendererMessage = {
+        requestId: newId,
+        data: toRaw(klona(data)),
+      };
+
+      if (!browserWindow || browserWindow.isDestroyed()) {
+        return reject(new Error("Browser window is destroyed"));
+      }
+
+      const cancel = on(newId, async (event, data) => {
+        // console.log('receiving event', event, data)
+        if (data.type === "end") {
+          cancel();
+          return resolve(data.data);
+        } else {
+          await listener?.(event, data);
+        }
+      });
+
+      // send the message
+      try {
+        browserWindow.webContents.send(channel, message);
+      } catch (e) {
+        logger().error(e);
+        logger().error(channel, message);
+        reject(e);
+      }
+    });
+  };
+
+  return {
+    send,
+    on,
+    execute,
+  };
+};
+
+export type UseMainAPI = ReturnType<typeof usePluginAPI>;

package/src/config.ts

@@ -0,0 +1,105 @@
+import { PipelabContext } from "./context";
+import path from "node:path";
+import { ensure } from "./utils/fs-extras";
+import fs from "node:fs/promises";
+import { useLogger } from "@pipelab/shared";
+import { configRegistry, Migrator } from "@pipelab/shared";
+
+export const getMigrator = <T>(name: string) => {
+  if (configRegistry[name]) {
+    return configRegistry[name] as Migrator<T>;
+  }
+
+  if (name.startsWith("pipeline-")) {
+    return configRegistry["pipeline"] as Migrator<T>;
+  }
+
+  return undefined;
+};
+
+export const setupConfigFile = async <T>(
+  name: string,
+  options: { context: PipelabContext; migrator?: Migrator<T> },
+) => {
+  const ctx = options.context;
+  const migrator = options.migrator || getMigrator<T>(name);
+
+  if (!migrator) {
+    throw new Error(
+      `No migrator found for configuration: ${name}. All managed files must have a migration schema.`,
+    );
+  }
+
+  const isAbsolutePath = path.isAbsolute(name);
+  const filesPath = isAbsolutePath ? name : ctx.getConfigPath(`${name}.json`);
+
+  await ensure(filesPath, JSON.stringify(migrator.defaultValue));
+
+  return {
+    setConfig: async (config: T) => {
+      const { logger } = useLogger();
+      try {
+        await fs.writeFile(filesPath, JSON.stringify(config));
+        return true;
+      } catch (e) {
+        logger().error(`Error saving config ${name}:`, e);
+        return false;
+      }
+    },
+    getConfig: async () => {
+      const { logger } = useLogger();
+      let content = undefined;
+      let originalJson: any = undefined;
+
+      try {
+        content = await fs.readFile(filesPath, "utf8");
+        if (content !== undefined) {
+          originalJson = JSON.parse(content);
+        }
+      } catch (e) {
+        logger().error(`Error reading or parsing config ${name}:`, e);
+      }
+
+      // console.log("content", content);
+
+      let json: any = undefined;
+      try {
+        json = await migrator.migrate(originalJson, {
+          debug: false,
+        });
+      } catch (e) {
+        logger().error(`Error migrating config ${name}:`, e);
+        json = migrator.defaultValue;
+      }
+
+      const originalVersion = originalJson?.version;
+      const newVersion = json?.version;
+
+      // Check if migration actually changed the version
+      if (originalVersion !== newVersion && content !== undefined) {
+        // Backup previous file before overwriting
+        const parsedPath = path.parse(filesPath);
+        const versionSuffix = originalVersion || "unknown";
+        const backupPath = path.join(parsedPath.dir, `${parsedPath.name}.${versionSuffix}.bak`);
+
+        try {
+          await fs.copyFile(filesPath, backupPath);
+          logger().info(`Backup created for ${name} at ${backupPath} (version ${versionSuffix})`);
+        } catch (e) {
+          logger().error(`Failed to create backup for ${name}:`, e);
+        }
+      }
+
+      // Save back migrated config if changed or if it's a new file
+      if (originalVersion !== newVersion || content === undefined) {
+        try {
+          await fs.writeFile(filesPath, JSON.stringify(json));
+        } catch (e) {
+          logger().error(`Error saving migrated config ${name}:`, e);
+        }
+      }
+
+      return json as T;
+    },
+  };
+};

package/src/context.ts

@@ -0,0 +1,53 @@
+import { join, dirname, resolve } from "node:path";
+import { homedir } from "node:os";
+import { fileURLToPath } from "node:url";
+import { existsSync } from "node:fs";
+
+const _dirname =
+  typeof __dirname !== "undefined"
+    ? __dirname
+    : typeof import.meta !== "undefined" && import.meta.url
+      ? dirname(fileURLToPath(import.meta.url))
+      : process.cwd();
+
+export const isDev = process.env.NODE_ENV === "development";
+
+/**
+ * Finds the monorepo root by looking for pnpm-workspace.yaml.
+ */
+function findProjectRoot(startDir: string): string | null {
+  let curr = startDir;
+  while (curr !== dirname(curr)) {
+    if (existsSync(join(curr, "pnpm-workspace.yaml"))) {
+      return curr;
+    }
+    curr = dirname(curr);
+  }
+  return null;
+}
+
+export const projectRoot = findProjectRoot(_dirname);
+
+export interface PipelabContextOptions {
+  userDataPath: string;
+}
+
+export class PipelabContext {
+  public readonly userDataPath: string;
+
+  constructor(options: PipelabContextOptions) {
+    this.userDataPath = options.userDataPath;
+  }
+
+  getPackagesPath(...subpaths: string[]) {
+    return join(this.userDataPath, "packages", ...subpaths);
+  }
+
+  getThirdPartyPath(...subpaths: string[]) {
+    return join(this.userDataPath, "thirdparty", ...subpaths);
+  }
+
+  getConfigPath(...subpaths: string[]) {
+    return join(this.userDataPath, "config", ...subpaths);
+  }
+}

package/src/handler-func.ts

@@ -0,0 +1,234 @@
+import { End } from "@pipelab/shared";
+import { ensureNodeJS, ensurePNPM } from "./utils/remote";
+import { Action, ActionRunner, Condition, ConditionRunner } from "./types/runner";
+import { InputsDefinition } from "@pipelab/shared";
+import { usePlugins } from "@pipelab/shared";
+import { isRequired } from "@pipelab/shared";
+import { mkdir, stat } from "node:fs/promises";
+import { PipelabContext } from "./context";
+import { useLogger } from "@pipelab/shared";
+import { BlockCondition } from "@pipelab/shared";
+import { HandleListenerSendFn } from "./handlers";
+import path from "node:path";
+const { join } = path;
+
+const checkParams = (definitionParams: InputsDefinition, elementParams: Record<string, string>) => {
+  // get a list of all required params
+  let expected = Object.keys(definitionParams);
+
+  const found: string[] = [];
+  // for each param in elementParams
+  for (const param of Object.keys(elementParams)) {
+    // if the param is in the expected list
+    if (expected.includes(param)) {
+      // add to found
+      found.push(param);
+      // remove from expected
+      expected = expected.filter((x) => x !== param);
+    } else {
+      // throw new Error('Unexpected param "' + param + '"')
+      console.warn('Unexpected param "' + param + '"');
+    }
+  }
+
+  for (const param of expected) {
+    if (isRequired(definitionParams[param])) {
+      throw new Error('Missing param "' + param + '"');
+    }
+  }
+};
+
+export const handleConditionExecute = async (
+  nodeId: string,
+  pluginId: string,
+  params: BlockCondition["params"],
+  cwd: string,
+  context: PipelabContext,
+): Promise<End<"condition:execute">> => {
+  const ctx = context;
+  const { plugins } = usePlugins();
+  const { logger } = useLogger();
+
+  const node = plugins.value
+    .find((plugin) => plugin.id === pluginId)
+    ?.nodes.find((node: any) => node.node.id === nodeId) as
+    | {
+        node: Condition;
+        runner: ConditionRunner<any>;
+      }
+    | undefined;
+
+  if (!node) {
+    return {
+      type: "error",
+      ipcError: "Node not found",
+    };
+  }
+
+  try {
+    const s = await stat(cwd);
+    if (!s.isDirectory()) {
+      throw new Error(`Execution directory "${cwd}" exists but is not a directory.`);
+    }
+  } catch (e: any) {
+    if (e.code === "ENOENT") {
+      await mkdir(cwd, { recursive: true });
+    } else {
+      throw e;
+    }
+  }
+
+  try {
+    checkParams(node.node.params, params);
+    const resolvedInputs = params; // await resolveConditionInputs(params, node.node, steps)
+
+    const result = await node.runner({
+      inputs: resolvedInputs,
+      log: (...args) => {
+        logger().info(`[${node.node.name}]`, ...args);
+      },
+      meta: {
+        definition: "",
+      },
+      setMeta: () => {
+        logger().info("set meta defined here");
+      },
+      cwd,
+      context: ctx,
+    });
+
+    return {
+      type: "success",
+      result: {
+        outputs: {},
+        value: result,
+      },
+    };
+  } catch (e) {
+    logger().error("Error in condition execution:", e);
+    return {
+      type: "error",
+      ipcError: String(e),
+    };
+  }
+};
+
+export const handleActionExecute = async (
+  nodeId: string,
+  pluginId: string,
+  params: Record<string, string>,
+  mainWindow: any | undefined,
+  send: HandleListenerSendFn<"action:execute">,
+  abortSignal: AbortSignal,
+  cwd: string,
+  cachePath: string,
+  context: PipelabContext,
+): Promise<End<"action:execute">> => {
+  const ctx = context;
+  const { plugins } = usePlugins();
+  const { logger } = useLogger();
+
+  mainWindow?.setProgressBar(1, {
+    mode: "indeterminate",
+  });
+
+  const node = plugins.value
+    .find((plugin) => plugin.id === pluginId)
+    ?.nodes.find((node: any) => node.node.id === nodeId) as
+    | {
+        node: Action;
+        runner: ActionRunner<any>;
+      }
+    | undefined;
+
+  if (!node) {
+    mainWindow?.setProgressBar(1, { mode: "normal" });
+    return {
+      type: "error",
+      ipcError: "Node not found",
+    };
+  }
+
+  const nodePath = await ensureNodeJS("24.14.1", { context: ctx });
+  const pnpm = await ensurePNPM("10.12.0", { context: ctx });
+
+  const outputs: Record<string, unknown> = {};
+
+  try {
+    try {
+      const s = await stat(cwd);
+      if (!s.isDirectory()) {
+        throw new Error(`Execution directory "${cwd}" exists but is not a directory.`);
+      }
+    } catch (e: any) {
+      if (e.code === "ENOENT") {
+        await mkdir(cwd, { recursive: true });
+      } else {
+        throw e;
+      }
+    }
+    checkParams(node.node.params, params);
+    const resolvedInputs = params; // await resolveActionInputs(params, node.node, steps)
+    logger().info("resolvedInputs", resolvedInputs);
+
+    await node.runner({
+      inputs: resolvedInputs,
+      log: (...args) => {
+        const decorator = `[${node.node.name}]`;
+        const logArgs = [decorator, ...args];
+        logger().info(...logArgs);
+        send({
+          type: "log",
+          data: {
+            decorator,
+            time: Date.now(),
+            message: args,
+          },
+        });
+      },
+      setOutput: (key: any, value: unknown) => {
+        outputs[key] = value;
+      },
+      meta: {
+        definition: "",
+      },
+      setMeta: () => {
+        logger().info("set meta defined here");
+      },
+      cwd: cwd,
+      paths: {
+        cache: cachePath,
+        node: nodePath,
+        pnpm,
+        modules: "",
+        userData: ctx.userDataPath,
+        thirdparty: ctx.getThirdPartyPath(),
+      },
+      browserWindow: mainWindow,
+      abortSignal,
+      context: ctx,
+    });
+
+    mainWindow?.setProgressBar(1, { mode: "normal" });
+
+    return {
+      type: "success",
+      result: { outputs, tmp: cwd },
+    };
+  } catch (e) {
+    logger().error("Error in action execution:", e);
+    mainWindow?.setProgressBar(1, { mode: "normal" });
+
+    // If aborted, throw AbortError to distinguish from actual errors
+    if (abortSignal.aborted) {
+      const abortError = new Error("Aborted");
+      abortError.name = "AbortError";
+      throw abortError;
+    }
+
+    return {
+      type: "error",
+      ipcError: String(e),
+    };
+  }
+};

package/src/handlers/agents.ts

@@ -0,0 +1,32 @@
+import { useAPI } from "../ipc-core";
+import { useLogger } from "@pipelab/shared";
+import { webSocketServer } from "../websocket-server";
+import { PipelabContext } from "../context";
+
+export const registerAgentsHandlers = (_context: PipelabContext) => {
+  const { handle } = useAPI();
+  const { logger } = useLogger();
+
+  handle("agents:get", async (event, { send }) => {
+    try {
+      const agents = webSocketServer.getAgents();
+
+      send({
+        type: "end",
+        data: {
+          type: "success",
+          result: { agents },
+        },
+      });
+    } catch (error) {
+      logger().error("Failed to get agents:", error);
+      send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: error instanceof Error ? error.message : "Failed to get agents",
+        },
+      });
+    }
+  });
+};

package/src/handlers/auth.ts

@@ -0,0 +1,95 @@
+import { supabase, useLogger, isSupabaseAvailable } from "@pipelab/shared";
+import { useAPI } from "../ipc-core";
+import { JsonFileStorage } from "../utils/storage";
+import { PipelabContext } from "../context";
+import { webSocketServer } from "../websocket-server";
+
+/**
+ * Registers authentication handlers for the CLI/system backend.
+ */
+export const registerAuthHandlers = (context: PipelabContext) => {
+  const { handle } = useAPI();
+  const { logger } = useLogger();
+
+  const supabaseAvailable = isSupabaseAvailable();
+  if (!supabaseAvailable) {
+    logger().warn("[Auth] Supabase is not available. Auth handlers will not be functional.");
+    return;
+  }
+
+  // Initialize the singleton Supabase client for the backend with the custom FileStorage
+  const client = supabase({
+    auth: {
+      storage: new JsonFileStorage("auth-session.json", context),
+      persistSession: true,
+      autoRefreshToken: true,
+      detectSessionInUrl: false,
+    },
+  });
+
+  handle("auth:getUser", async (_, { send }) => {
+    try {
+      const { data, error } = await client.auth.getUser();
+      if (error) {
+        return send({ type: "end", data: { type: "success", result: { user: null } } });
+      }
+      return send({ type: "end", data: { type: "success", result: { user: data.user } } });
+    } catch (e) {
+      return send({ type: "end", data: { type: "success", result: { user: null } } });
+    }
+  });
+
+  handle("auth:signInWithPassword", async (_, { value, send }) => {
+    const { email, password } = value;
+    logger().info("[Auth] signInWithPassword:", email);
+    const result = await client.auth.signInWithPassword({ email, password });
+    return send({ type: "end", data: { type: "success", result: result as any } });
+  });
+
+  handle("auth:signUp", async (_, { value, send }) => {
+    const { email, password } = value;
+    logger().info("[Auth] signUp:", email);
+    const result = await client.auth.signUp({ email, password });
+    return send({ type: "end", data: { type: "success", result: result as any } });
+  });
+
+  handle("auth:signOut", async (_, { send }) => {
+    logger().info("[Auth] signOut");
+    await client.auth.signOut();
+    return send({ type: "end", data: { type: "success", result: undefined } });
+  });
+
+  handle("auth:resetPasswordForEmail", async (_, { value, send }) => {
+    const { email } = value;
+    logger().info("[Auth] resetPasswordForEmail:", email);
+    const { error } = await client.auth.resetPasswordForEmail(email);
+    return send({ type: "end", data: { type: "success", result: { error } } });
+  });
+
+  handle("auth:invoke", async (_, { value, send }) => {
+    const { name, options } = value;
+    logger().info("[Auth] invoke function:", name);
+    try {
+      const result = await client.functions.invoke(name, options);
+      return send({ type: "end", data: { type: "success", result } });
+    } catch (e) {
+      return send({
+        type: "end",
+        data: {
+          type: "error",
+          ipcError: e instanceof Error ? e.message : "Edge Function invocation failed",
+        },
+      });
+    }
+  });
+
+  // Maintain logs for the backend state change and broadcast to all clients
+  client.auth.onAuthStateChange(async (event, session) => {
+    logger().info(
+      "[Auth] State changed, broadcasting:",
+      event,
+      session?.user?.email || "anonymous",
+    );
+    webSocketServer.broadcast("auth:getUser" as any, { user: session?.user || null } as any);
+  });
+};