@pipelab/core-node 1.0.1-beta.23

package/src/utils.ts

@@ -0,0 +1,258 @@
+import { nanoid } from "nanoid";
+import { usePlugins } from "@pipelab/shared";
+import { RendererPluginDefinition } from "@pipelab/shared";
+import { downloadFile, DownloadHooks } from "./utils/fs-extras";
+import { access, chmod, mkdir, rm, writeFile, readdir, cp } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { isDev, projectRoot, PipelabContext } from "./context";
+import { constants, existsSync } from "node:fs";
+import { processGraph } from "@pipelab/shared";
+import { handleActionExecute } from "./handler-func";
+import { useLogger } from "@pipelab/shared";
+import { BuildHistoryStorage } from "./handlers/build-history";
+import type { BuildHistoryEntry } from "@pipelab/shared";
+import type { Variable } from "@pipelab/shared";
+
+import { ensure, generateTempFolder, extractTarGz, extractZip, zipFolder } from "./utils/fs-extras";
+import { fetchPipelabAsset, fetchPipelabPlugin } from "./utils/remote";
+import { setupConfigFile } from "./config";
+import { AppConfig } from "@pipelab/shared";
+
+export const getFinalPlugins = () => {
+  const { plugins } = usePlugins();
+  // console.log('plugins.value', plugins.value)
+
+  const finalPlugins: RendererPluginDefinition[] = [];
+
+  for (const plugin of plugins.value) {
+    const finalNodes = [];
+
+    const transformUrl = (url: string) => {
+      if (url.startsWith("file://")) {
+        return url.replace("file://", "media://");
+      }
+      return url;
+    };
+
+    const finalIcon =
+      plugin.icon?.type === "image"
+        ? {
+            ...plugin.icon,
+            image: transformUrl(plugin.icon.image),
+          }
+        : plugin.icon;
+
+    for (const nodeDef of plugin.nodes) {
+      const node = nodeDef.node;
+      finalNodes.push({
+        ...nodeDef,
+        node: {
+          ...node,
+          icon: transformUrl(node.icon),
+        },
+      });
+    }
+
+    finalPlugins.push({
+      ...plugin,
+      icon: finalIcon,
+      nodes: finalNodes,
+    });
+  }
+
+  return finalPlugins;
+};
+
+import { ensureNodeJS, ensurePNPM } from "./utils/remote";
+
+export const executeGraphWithHistory = async ({
+  graph,
+  variables,
+  projectName,
+  projectPath,
+  pipelineId,
+  onNodeEnter,
+  onNodeExit,
+  onLog,
+  abortSignal,
+  mainWindow,
+  cachePath,
+  context,
+}: {
+  graph: any;
+  variables: Variable[];
+  projectName: string;
+  projectPath: string;
+  pipelineId: string;
+  onNodeEnter?: (node: any) => void;
+  onNodeExit?: (node: any) => void;
+  onLog?: (data: any, node?: any) => void;
+  abortSignal: AbortSignal;
+  mainWindow?: any;
+  cachePath: string;
+  context: PipelabContext;
+}) => {
+  const ctx = context;
+  const buildHistoryStorage = new BuildHistoryStorage(ctx);
+  const buildId = nanoid();
+  const startTime = Date.now();
+
+  // Save initial build history entry
+  const now = Date.now();
+  const initialEntry: BuildHistoryEntry = {
+    id: buildId,
+    projectName,
+    projectPath,
+    pipelineId,
+    startTime,
+    status: "running",
+    logs: [],
+    steps: [],
+    totalSteps: graph.length,
+    completedSteps: 0,
+    failedSteps: 0,
+    cancelledSteps: 0,
+    createdAt: now,
+    updatedAt: now,
+  };
+  const shouldDisableHistory = process.env.PIPELAB_DISABLE_HISTORY === "true";
+
+  if (!shouldDisableHistory) {
+    await buildHistoryStorage.save(initialEntry);
+  }
+
+  const logs: any[] = [];
+  const sandboxPath = await generateTempFolder(cachePath);
+  const { logger } = useLogger();
+
+  // Ensure all plugins used in the graph are downloaded and registered
+  const { registerPlugins, plugins: registeredPlugins } = usePlugins();
+  const pluginIds = new Set(
+    graph.map((node: any) => node.origin?.pluginId).filter(Boolean),
+  ) as Set<string>;
+
+  for (const pluginId of pluginIds) {
+    const isRegistered = registeredPlugins.value.some((p) => p.id === pluginId);
+    if (!isRegistered) {
+      logger().info(`[Runner] Plugin "${pluginId}" not found, attempting to load...`);
+      try {
+        const packageName = `@pipelab/plugin-${pluginId}`;
+
+        const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, "latest", {
+          context: ctx,
+        });
+        const pluginModule = await import(pathToFileURL(entryPoint).href);
+        const pluginDefinition = pluginModule.default;
+
+        registerPlugins([pluginDefinition]);
+        logger().info(`[Runner] Plugin "${pluginId}" loaded and registered successfully`);
+      } catch (e) {
+        logger().error(`[Runner] Failed to load or register plugin "${pluginId}":`, e);
+      }
+    }
+  }
+
+  logger().info(`[Sandbox] Execution sandbox created at: ${sandboxPath}`);
+  const settingsFile = await setupConfigFile<AppConfig>("settings", { context: ctx });
+  const config = await settingsFile.getConfig();
+  const shouldCleanup = config?.clearTemporaryFoldersOnPipelineEnd ?? true;
+
+  try {
+    const result = await processGraph({
+      graph,
+      definitions: getFinalPlugins(),
+      variables,
+      steps: {},
+      context: {},
+      onExecuteItem: async (node, params, steps) => {
+        if (node.type === "action") {
+          return await handleActionExecute(
+            node.origin.nodeId,
+            node.origin.pluginId,
+            params,
+            mainWindow,
+            async (data) => {
+              if (data.type === "log") {
+                const logEntry = {
+                  type: data.type,
+                  message: data.data.message,
+                  timestamp: data.data.time,
+                  nodeUid: node?.uid,
+                };
+                logs.push(logEntry);
+                onLog?.(data, node);
+              }
+            },
+            abortSignal,
+            sandboxPath,
+            cachePath,
+            ctx,
+          );
+        }
+        throw new Error(`Execution of node type ${node.type} not implemented in utils.ts`);
+      },
+      onNodeEnter: (node) => {
+        onNodeEnter?.(node);
+      },
+      onNodeExit: (node) => {
+        onNodeExit?.(node);
+      },
+      abortSignal,
+    });
+
+    const endTime = Date.now();
+    if (!shouldDisableHistory) {
+      await buildHistoryStorage.update(
+        buildId,
+        {
+          status: "completed",
+          endTime,
+          duration: endTime - startTime,
+          output: result.steps,
+          logs,
+        },
+        pipelineId,
+      );
+    }
+
+    return { result, buildId };
+  } catch (error) {
+    const endTime = Date.now();
+    const isCanceled = error instanceof Error && error.name === "AbortError";
+
+    if (!shouldDisableHistory) {
+      await buildHistoryStorage.update(
+        buildId,
+        {
+          status: isCanceled ? "cancelled" : "failed",
+          endTime,
+          duration: endTime - startTime,
+          error: {
+            message: error instanceof Error ? error.message : String(error),
+            stack: error instanceof Error ? error.stack : undefined,
+            timestamp: endTime,
+          },
+          logs,
+        },
+        pipelineId,
+      );
+    }
+
+    throw error;
+  } finally {
+    // Apply retention policy regardless of outcome
+    if (!shouldDisableHistory) {
+      // Don't await, let it run in the background
+      buildHistoryStorage.applyRetentionPolicy();
+    }
+    
+    if (shouldCleanup) {
+      try {
+        await rm(sandboxPath, { recursive: true, force: true });
+      } catch (e) {
+        console.warn(`Failed to cleanup sandbox at ${sandboxPath}:`, e);
+      }
+    }
+  }
+};

package/src/websocket-server.ts

@@ -0,0 +1,288 @@
+import { WebSocketServer as WSWebSocketServer, WebSocket as WSWebSocket } from "ws";
+import http, { IncomingMessage } from "node:http";
+import { nanoid } from "nanoid";
+import { useAPI } from "./ipc-core";
+import { useLogger } from "@pipelab/shared";
+import {
+  WebSocketServerConfig,
+  WebSocketServerEvents,
+  WebSocketConnectionState,
+  WebSocketError,
+  WebSocketMessage,
+  isWebSocketRequestMessage,
+  Agent,
+  Channels,
+  Events,
+} from "@pipelab/shared";
+import { websocketPort } from "@pipelab/constants";
+
+export interface ConnectedClient {
+  id: string;
+  name: string;
+  ws: WSWebSocket;
+  connectedAt: number;
+}
+
+export class WebSocketServer {
+  private wss: WSWebSocketServer | null = null;
+  private server: import("http").Server | null = null;
+  private isReady = false;
+  private readyResolve: (() => void) | null = null;
+  private connectionState: WebSocketConnectionState = "disconnected";
+  private clients: Map<WSWebSocket, ConnectedClient> = new Map();
+  private lastBroadcasts: Map<Channels, string> = new Map();
+
+  async start(port: number = websocketPort, existingServer?: import("http").Server): Promise<void> {
+    const { logger } = useLogger();
+
+    return new Promise((resolve, reject) => {
+      try {
+        logger().info("Starting WebSocket server on port", port);
+
+        // Create HTTP server for WebSocket if not provided
+        const server = existingServer || http.createServer();
+        this.server = server;
+
+        // Create WebSocket server
+        this.wss = new WSWebSocketServer({ server });
+
+        this.wss.on("connection", (ws: WSWebSocket, request: IncomingMessage) => {
+          const clientId = nanoid();
+          const clientName = `Agent ${clientId.substring(0, 4)}`;
+          const connectedAt = Date.now();
+
+          const client: ConnectedClient = {
+            id: clientId,
+            name: clientName,
+            ws,
+            connectedAt,
+          };
+
+          this.clients.set(ws, client);
+          console.log("WebSocket client connected", {
+            id: clientId,
+            name: clientName,
+            url: request.url,
+          });
+
+          // Replay last known state for each channel that has been broadcasted
+          for (const messageStr of this.lastBroadcasts.values()) {
+            try {
+              ws.send(messageStr);
+            } catch (error) {
+              logger().error("[WebSocket] Failed to replay sticky event to new client:", error);
+            }
+          }
+
+          ws.on("message", (data: Buffer) => {
+            try {
+              const message = JSON.parse(data.toString());
+              logger().debug("Received WebSocket message:", message);
+              this.handleWebSocketMessage(ws, message);
+            } catch (error) {
+              logger().error("Failed to parse WebSocket message:", error);
+              this.sendError(ws, undefined, "Invalid message format");
+            }
+          });
+
+          ws.on("close", (code: number, reason: Buffer) => {
+            const client = this.clients.get(ws);
+            if (client) {
+              logger().info("WebSocket client disconnected", {
+                id: client.id,
+                name: client.name,
+                code,
+                reason: reason.toString(),
+              });
+              this.clients.delete(ws);
+            }
+          });
+
+          ws.on("error", (error: Error) => {
+            logger().error("WebSocket client error:", error);
+          });
+        });
+
+        this.wss.on("error", (error: Error) => {
+          logger().error("WebSocket server error:", error);
+        });
+
+        if (!server.listening) {
+          server.listen(port, () => {
+            this.connectionState = "connected";
+            logger().info(`WebSocket server listening on port ${port}`);
+            this.isReady = true;
+            if (this.readyResolve) {
+              this.readyResolve();
+            }
+            resolve();
+          });
+        } else {
+          this.connectionState = "connected";
+          logger().info(`WebSocket server attached to existing listening server`);
+          this.isReady = true;
+          if (this.readyResolve) {
+            this.readyResolve();
+          }
+          resolve();
+        }
+
+        server.on("error", (error: Error) => {
+          this.connectionState = "error";
+          logger().error("WebSocket server HTTP error:", error);
+          reject(new WebSocketError(`Server error: ${error.message}`));
+        });
+      } catch (error) {
+        this.connectionState = "error";
+        logger().error("Failed to start WebSocket server:", error);
+        reject(
+          new WebSocketError(
+            `Failed to start server: ${error instanceof Error ? error.message : "Unknown error"}`,
+          ),
+        );
+      }
+    });
+  }
+
+  private async handleWebSocketMessage(ws: WSWebSocket, message: WebSocketMessage) {
+    const { logger } = useLogger();
+    const { processWebSocketMessage } = useAPI();
+
+    try {
+      if (isWebSocketRequestMessage(message)) {
+        logger().debug("Processing WebSocket request message:", {
+          channel: message.channel,
+          requestId: message.requestId,
+          hasData: !!message.data,
+        });
+
+        logger().debug("Routing message to handler:", message.channel);
+        await processWebSocketMessage(ws, message.channel, message);
+        logger().debug("Handler processed message successfully");
+      } else {
+        logger().warn("Invalid message format received:", {
+          type: message.type,
+          requestId: message.requestId,
+          hasError: "error" in message,
+        });
+        this.sendError(
+          ws,
+          message.requestId,
+          "Invalid message format: missing channel or requestId",
+        );
+      }
+    } catch (error) {
+      logger().error("Error processing WebSocket message:", error);
+      this.sendError(
+        ws,
+        message.requestId,
+        error instanceof Error ? error.message : "Unknown error",
+      );
+    }
+  }
+
+  private sendError(
+    ws: WSWebSocket,
+    requestId: string | undefined,
+    errorMessage: string,
+    code?: string,
+  ) {
+    try {
+      const errorResponse: any = {
+        type: "error",
+        requestId,
+        error: errorMessage,
+        ...(code && { code }),
+      };
+      ws.send(JSON.stringify(errorResponse));
+    } catch (error) {
+      console.error("Failed to send error response:", error);
+    }
+  }
+
+  async stop(): Promise<void> {
+    const { logger } = useLogger();
+
+    return new Promise((resolve) => {
+      this.isReady = false;
+      this.connectionState = "disconnected";
+
+      if (this.wss) {
+        this.wss.close(() => {
+          logger().info("WebSocket server closed");
+          resolve();
+        });
+      } else {
+        resolve();
+      }
+
+      if (this.server) {
+        this.server.close();
+      }
+    });
+  }
+
+  waitForReady(): Promise<void> {
+    if (this.isReady) {
+      return Promise.resolve();
+    }
+
+    return new Promise((resolve) => {
+      this.readyResolve = resolve;
+    });
+  }
+
+  isServerReady(): boolean {
+    return this.isReady;
+  }
+
+  getConnectionState(): WebSocketConnectionState {
+    return this.connectionState;
+  }
+
+  getAgents(): Agent[] {
+    return Array.from(this.clients.values()).map((client) => ({
+      id: client.id,
+      name: client.name,
+      isSelf: false, // This will be set by the client itself when receiving the list
+      connectedAt: client.connectedAt,
+    }));
+  }
+
+  getClient(ws: WSWebSocket): ConnectedClient | undefined {
+    return this.clients.get(ws);
+  }
+
+  /**
+   * Broadcasts a message to all connected clients.
+   * Useful for push notifications like auth state changes.
+   */
+  broadcast<KEY extends Channels>(channel: KEY, data: Events<KEY>): void {
+    const { logger } = useLogger();
+
+    const message = {
+      type: "event",
+      channel,
+      data,
+    };
+
+    const messageStr = JSON.stringify(message);
+    this.lastBroadcasts.set(channel, messageStr);
+
+    if (!this.wss || this.clients.size === 0) return;
+    logger().debug(`[WebSocket] Broadcasting to all clients on channel: ${channel}`);
+
+    for (const client of this.clients.values()) {
+      try {
+        if (client.ws.readyState === WSWebSocket.OPEN) {
+          client.ws.send(messageStr);
+        }
+      } catch (error) {
+        logger().error(`[WebSocket] Failed to broadcast to client ${client.id}:`, error);
+      }
+    }
+  }
+}
+
+// Export singleton instance
+export const webSocketServer = new WebSocketServer();

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../tsconfig/node.json",
+  "compilerOptions": {
+    "moduleResolution": "bundler",
+    "outDir": "dist",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "baseUrl": ".",
+    "paths": {
+      "@pipelab/shared": ["../shared/src"],
+      "@pipelab/constants": ["../constants/src/index.ts"],
+      "@pipelab/*": ["../shared/src/libs/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"],
+  "references": [{ "path": "../constants" }, { "path": "../shared" }]
+}