@pipelab/core-node 1.0.0-beta.0

package/src/runner.ts

@@ -0,0 +1,160 @@
+import { executeGraphWithHistory } from "./utils";
+import { setupConfigFile } from "./config";
+import { isDev, PipelabContext } from "./context";
+import { readFile, access, writeFile, mkdir } from "node:fs/promises";
+import { resolve, isAbsolute, join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import { savedFileMigrator } from "@pipelab/shared";
+import type { AppConfig } from "@pipelab/shared";
+import { registerMigrationHandlers } from "./migrations";
+import { registerAllHandlers } from "./handlers/index";
+
+export interface RunOptions {
+  userData?: string;
+  variables?: string;
+  output?: string;
+  cloud?: boolean;
+}
+
+export async function runPipelineCommand(file: string, options: RunOptions, version: string) {
+  const pipelinePath = isAbsolute(file) ? file : resolve(process.cwd(), file);
+
+  try {
+    await access(pipelinePath);
+  } catch (e) {
+    throw new Error(`Pipeline file not found at ${pipelinePath}`);
+  }
+
+  const pipelineContent = await readFile(pipelinePath, "utf-8");
+  const pipeline = JSON.parse(pipelineContent);
+
+  let finalPipeline = pipeline;
+  if (pipeline.version) {
+    finalPipeline = await savedFileMigrator.migrate(pipeline);
+  }
+
+  const {
+    graph: rawGraph,
+    variables: pipelineVariables,
+    projectName,
+    projectPath,
+    pipelineId,
+    canvas,
+    name,
+  } = finalPipeline;
+
+  const graph = rawGraph || canvas?.blocks;
+
+  if (!graph) {
+    throw new Error("Pipeline does not contain a valid graph or canvas.blocks");
+  }
+
+  let vars = pipelineVariables || finalPipeline.variables || [];
+  if (options.variables) {
+    vars = JSON.parse(options.variables);
+  }
+
+  const effectiveProjectName = projectName || name || "CLI Run";
+  const effectiveProjectPath = projectPath || process.cwd();
+  const effectivePipelineId = pipelineId || "cli-run";
+
+  console.log(`Executing pipeline: ${effectiveProjectName} (${effectivePipelineId})`);
+
+  if (!options.userData) throw new Error("userDataPath is required for runPipelineCommand");
+  const context = new PipelabContext({
+    userDataPath: options.userData,
+  });
+
+  await registerAllHandlers({ version, context });
+  registerMigrationHandlers(context);
+
+  const settings = await setupConfigFile<AppConfig>("settings", { context });
+  const config = await settings.getConfig();
+  const cachePath = join(context.userDataPath, "cache");
+  await mkdir(cachePath, { recursive: true });
+
+  const abortController = new AbortController();
+
+  let supabase: any;
+  const cloudRunId = process.env.CLOUD_RUN_ID;
+  if (options.cloud && cloudRunId) {
+    const { createClient } = await import("@supabase/supabase-js");
+    supabase = createClient(
+      process.env.SUPABASE_URL!,
+      process.env.SUPABASE_SERVICE_ROLE_KEY!
+    );
+    console.log(`Cloud mode enabled. Streaming logs for run: ${cloudRunId}`);
+  }
+
+  try {
+    const { result, buildId } = await executeGraphWithHistory({
+      graph,
+      variables: vars,
+      projectName: effectiveProjectName,
+      projectPath: effectiveProjectPath,
+      pipelineId: effectivePipelineId,
+      cachePath: cachePath,
+      onNodeEnter: (node) => {
+        console.log(`[ENTER] ${node.name} (${node.uid})`);
+        if (supabase) {
+          supabase.from("cloud_run_logs").insert({
+            run_id: cloudRunId,
+            message: `[ENTER] ${node.name} (${node.uid})`,
+            node_uid: node.uid,
+            type: "node-enter"
+          }).then();
+        }
+      },
+      onNodeExit: (node) => {
+        console.log(`[EXIT] ${node.name} (${node.uid})`);
+        if (supabase) {
+          supabase.from("cloud_run_logs").insert({
+            run_id: cloudRunId,
+            message: `[EXIT] ${node.name} (${node.uid})`,
+            node_uid: node.uid,
+            type: "node-exit"
+          }).then();
+        }
+      },
+      onLog: (data, node) => {
+        if (data.type === "log") {
+          const message = data.data.message.join(" ");
+          console.log(`[LOG] ${message}`);
+          if (supabase) {
+            supabase.from("cloud_run_logs").insert({
+              run_id: cloudRunId,
+              message: message,
+              node_uid: node?.uid,
+              type: "log"
+            }).then();
+          }
+        }
+      },
+      abortSignal: abortController.signal,
+      context,
+    });
+
+    console.log(`Pipeline execution finished. Build ID: ${buildId}`);
+
+    if (supabase) {
+      await supabase.from("cloud_runs").update({ status: "success" }).eq("id", cloudRunId);
+    }
+
+    if (options.output) {
+      const outputPath = isAbsolute(options.output)
+        ? options.output
+        : resolve(process.cwd(), options.output);
+
+      await mkdir(dirname(outputPath), { recursive: true });
+      await writeFile(outputPath, JSON.stringify(result, null, 2), "utf-8");
+      console.log(`Result saved to ${outputPath}`);
+    }
+
+    return result;
+  } catch (e) {
+    if (supabase) {
+      await supabase.from("cloud_runs").update({ status: "failed" }).eq("id", cloudRunId);
+    }
+    throw e;
+  }
+}

package/src/server.ts

@@ -0,0 +1,99 @@
+import {
+  ensurePNPM,
+  PipelabContext,
+  isDev,
+  fetchPipelabAsset,
+  registerAllHandlers,
+  webSocketServer,
+} from "./index";
+import { getUiDevServerMissingWarning, uiDevPort } from "@pipelab/constants";
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import http from "http";
+import handler from "serve-handler";
+import { registerMigrationHandlers } from "./migrations";
+
+export interface ServeOptions {
+  port: string | number;
+  userData?: string;
+  nodePath?: string;
+  pnpmPath?: string;
+}
+
+export const sendStartupProgress = (message: string) => {
+  console.log(`[Startup Progress] ${message}`);
+  webSocketServer.broadcast("startup:progress", {
+    type: "progress",
+    data: { message },
+  });
+};
+
+export const sendStartupReady = () => {
+  console.log(`[Startup Progress] Ready!`);
+  webSocketServer.broadcast("startup:progress", {
+    type: "ready",
+  });
+};
+
+export async function serveCommand(options: ServeOptions, version: string, _dirname: string) {
+  if (!options.userData) throw new Error("userDataPath is required for serveCommand");
+  const context = new PipelabContext({
+    userDataPath: options.userData,
+  });
+
+  let rawAssetFolder: string | undefined;
+  if (!isDev) {
+    rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", "latest", { context });
+  }
+
+  const server = http.createServer(async (request, response) => {
+    if (isDev) {
+      response.writeHead(200, { "Content-Type": "text/html" });
+      response.end(`
+        <html>
+          <body style="font-family: sans-serif; display: flex; flex-direction: column; align-items: center; justify-content: center; height: 100vh; margin: 0; background: #0f172a; color: #f8fafc;">
+            <h1 style="color: #38bdf8;">Pipelab Dev Mode</h1>
+            <p>The CLI server is running (API/WebSocket), but the UI is not served here in development.</p>
+            <p>Please open the UI through its own dev server (usually <a href="http://localhost:5173" style="color: #38bdf8;">http://localhost:5173</a>).</p>
+          </body>
+        </html>
+      `);
+      return;
+    }
+
+    if (!rawAssetFolder || !existsSync(rawAssetFolder)) {
+      response.writeHead(404, { "Content-Type": "text/plain" });
+      response.end(
+        `Error: UI directory not found at ${rawAssetFolder}.\n` +
+        "Please run 'pnpm build' in apps/ui to generate the distribution.",
+      );
+      return;
+    }
+
+    return handler(request, response, {
+      public: rawAssetFolder,
+    });
+  });
+
+  console.log(`Starting Pipelab server on port ${options.port}...`);
+  if (isDev) {
+    console.info(getUiDevServerMissingWarning());
+    console.log(`UI available at http://localhost:${uiDevPort}`);
+  } else {
+    console.log(`UI available at http://localhost:${options.port}`);
+  }
+
+  // Start the server EARLY so the UI can connect and receive progress updates
+  await webSocketServer.start(Number(options.port), server);
+
+  await registerAllHandlers({
+    version,
+    context,
+  });
+  registerMigrationHandlers(context);
+
+  sendStartupReady();
+
+  return server;
+}

package/src/types/runner.ts

@@ -0,0 +1,101 @@
+import type { BrowserWindow } from "electron";
+import type { PipelabContext } from "../context";
+import type {
+  Action,
+  Condition,
+  Loop,
+  Expression,
+  Event,
+  SetOutputActionFn,
+  SetOutputLoopFn,
+  SetOutputExpressionFn,
+  ExtractInputsFromAction,
+  ExtractInputsFromCondition,
+  ExtractInputsFromLoop,
+  ExtractInputsFromEvent,
+  ExtractInputsFromExpression,
+} from "@pipelab/shared";
+
+export type {
+  Action,
+  Condition,
+  Loop,
+  Expression,
+  Event,
+  SetOutputActionFn,
+  SetOutputLoopFn,
+  SetOutputExpressionFn,
+  ExtractInputsFromAction,
+  ExtractInputsFromCondition,
+  ExtractInputsFromLoop,
+  ExtractInputsFromEvent,
+  ExtractInputsFromExpression,
+};
+
+export type RunnerCallbackFnArgument = {
+  done: () => void;
+  id: string;
+  log: (...args: Parameters<(typeof console)["log"]>) => void;
+};
+
+export type ActionRunnerData<ACTION extends Action> = {
+  log: typeof console.log;
+  setOutput: SetOutputActionFn<ACTION>;
+  inputs: ExtractInputsFromAction<ACTION>;
+  setMeta: (callback: (data: ACTION["meta"]) => ACTION["meta"]) => void;
+  meta: ACTION["meta"];
+  cwd: string;
+  paths: {
+    cache: string;
+    pnpm: string;
+    node: string;
+    userData: string;
+    modules: string;
+    thirdparty: string;
+  };
+  browserWindow: BrowserWindow;
+  abortSignal: AbortSignal;
+  context: PipelabContext;
+};
+
+export type ActionRunner<ACTION extends Action> = (data: ActionRunnerData<ACTION>) => Promise<void>;
+
+export type ConditionRunner<CONDITION extends Condition> = (data: {
+  log: typeof console.log;
+  inputs: ExtractInputsFromCondition<CONDITION>;
+  setMeta: (callback: (data: CONDITION["meta"]) => CONDITION["meta"]) => void;
+  meta: CONDITION["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<boolean>;
+
+export type LoopRunner<LOOP extends Loop> = (data: {
+  log: typeof console.log;
+  setOutput: SetOutputLoopFn<LOOP>;
+  inputs: ExtractInputsFromLoop<LOOP>;
+  setMeta: (callback: (data: LOOP["meta"]) => LOOP["meta"]) => void;
+  meta: LOOP["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<"step" | "exit">;
+
+export type ExpressionRunner<EXPRESSION extends Expression> = (data: {
+  log: typeof console.log;
+  setOutput: SetOutputExpressionFn<EXPRESSION>;
+  inputs: ExtractInputsFromExpression<EXPRESSION>;
+  setMeta: (callback: (data: EXPRESSION["meta"]) => EXPRESSION["meta"]) => void;
+  meta: EXPRESSION["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<string>;
+
+export type EventRunner<EVENT extends Event> = (data: {
+  log: typeof console.log;
+  inputs: ExtractInputsFromEvent<EVENT>;
+  setMeta: (callback: (data: EVENT["meta"]) => EVENT["meta"]) => void;
+  meta: EVENT["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<void>;
+
+export type Runner = ActionRunner<any> | LoopRunner<any> | EventRunner<any> | ConditionRunner<any>;

package/src/utils/fs-extras.ts

@@ -0,0 +1,211 @@
+import { mkdir, createWriteStream, createReadStream } from "node:fs";
+import { execa, Options, Subprocess } from "execa";
+import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod, stat, readdir } from "node:fs/promises";
+import { join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import tar from "tar";
+import yauzl from "yauzl";
+import archiver from "archiver";
+import { pipeline } from "node:stream/promises";
+
+/**
+ * Ensures a directory exists and a file is created with default content if missing.
+ */
+export const ensure = async (filesPath: string, defaultContent = "{}") => {
+  await mkdirP(dirname(filesPath), { recursive: true });
+  try {
+    await access(filesPath);
+  } catch {
+    await writeFile(filesPath, defaultContent);
+  }
+};
+
+/**
+ * Generates a unique temporary folder.
+ */
+export const generateTempFolder = async (base?: string) => {
+  const targetBase = base || tmpdir();
+  await mkdirP(targetBase, { recursive: true });
+  const realPath = await realpath(targetBase);
+  const tempFolder = await mkdtemp(join(realPath, "pipelab-"));
+  return tempFolder;
+};
+
+/**
+ * Extracts a .tar.gz archive.
+ */
+export async function extractTarGz(archivePath: string, destinationDir: string): Promise<void> {
+  await mkdirP(destinationDir, { recursive: true });
+  await tar.x({
+    file: archivePath,
+    cwd: destinationDir,
+  });
+}
+
+/**
+ * Extracts a .zip archive.
+ */
+export async function extractZip(archivePath: string, destinationDir: string): Promise<void> {
+  await mkdirP(destinationDir, { recursive: true });
+  return new Promise((resolve, reject) => {
+    yauzl.open(archivePath, { lazyEntries: true }, (err, zipfile) => {
+      if (err || !zipfile) return reject(err || new Error("Could not open zip file"));
+      zipfile.on("error", reject);
+      zipfile.readEntry();
+      zipfile.on("entry", (entry) => {
+        const entryPath = join(destinationDir, entry.fileName);
+        if (/\/$/.test(entry.fileName)) {
+          mkdirP(entryPath, { recursive: true })
+            .then(() => zipfile.readEntry())
+            .catch(reject);
+        } else {
+          mkdirP(dirname(entryPath), { recursive: true })
+            .then(() => {
+              zipfile.openReadStream(entry, (err, readStream) => {
+                if (err || !readStream)
+                  return reject(err || new Error("Could not open read stream"));
+                readStream.on("error", reject);
+                const writeStream = createWriteStream(entryPath);
+                writeStream.on("error", reject);
+                writeStream.on("close", () => zipfile.readEntry());
+                readStream.pipe(writeStream);
+              });
+            })
+            .catch(reject);
+        }
+      });
+      zipfile.on("end", () => resolve());
+    });
+  });
+}
+
+/**
+ * Zips a folder.
+ */
+export const zipFolder = async (
+  from: string,
+  to: string,
+  log: typeof console.log = console.log,
+) => {
+  const output = createWriteStream(to);
+  const archive = archiver("zip", { zlib: { level: 9 } });
+  return new Promise<string>((resolve, reject) => {
+    output.on("close", () => {
+      log(archive.pointer() + " total bytes");
+      resolve(to);
+    });
+    archive.on("error", reject);
+    archive.pipe(output);
+    archive.directory(from, false);
+    archive.finalize();
+  });
+};
+
+/**
+ * Downloads a file with progress tracking.
+ */
+export interface DownloadHooks {
+  onProgress?: (data: { progress: number; downloadedSize: number }) => void;
+}
+
+export const downloadFile = async (
+  url: string,
+  localPath: string,
+  hooks?: DownloadHooks,
+  abortSignal?: AbortSignal,
+): Promise<void> => {
+  const response = await fetch(url, { signal: abortSignal });
+  if (!response.ok) throw new Error(`Failed to fetch file: ${response.statusText}`);
+  const contentLength = response.headers.get("content-length");
+  if (!contentLength) throw new Error("Content-Length header is missing");
+  const totalSize = parseInt(contentLength, 10);
+  let downloadedSize = 0;
+  const fileStream = createWriteStream(localPath);
+  const progressStream = new TransformStream({
+    transform(chunk, controller) {
+      downloadedSize += chunk.length;
+      const progress = (downloadedSize / totalSize) * 100;
+      hooks?.onProgress?.({ progress, downloadedSize });
+      controller.enqueue(chunk);
+    },
+  });
+  const readable = response.body?.pipeThrough(progressStream);
+  if (!readable) throw new Error("Failed to create a readable stream");
+  await pipeline(readable, fileStream);
+};
+
+export const runWithLiveLogs = async (
+  command: string,
+  args: string[],
+  execaOptions: Options,
+  log: typeof console.log,
+  hooks?: {
+    onStdout?: (data: string, subprocess: Subprocess) => void;
+    onStderr?: (data: string, subprocess: Subprocess) => void;
+    onExit?: (code: number) => void;
+    onCreated?: (subprocess: Subprocess) => void;
+  },
+  abortSignal?: AbortSignal,
+): Promise<void> => {
+  const subprocess = execa(command, args, {
+    ...execaOptions,
+    stdout: "pipe",
+    stderr: "pipe",
+    stdin: "pipe",
+    env: {
+      ...process.env,
+      ...execaOptions.env,
+      TERM: "xterm-256color",
+      FORCE_STDERR_LOGGING: "1",
+    },
+    cancelSignal: abortSignal,
+  });
+
+  hooks?.onCreated?.(subprocess);
+
+  subprocess.stdout?.on("data", (data: Buffer) => {
+    hooks?.onStdout?.(data.toString(), subprocess);
+  });
+
+  subprocess.stderr?.on("data", (data: Buffer) => {
+    hooks?.onStderr?.(data.toString(), subprocess);
+  });
+
+  try {
+    const { exitCode } = await subprocess;
+    hooks?.onExit?.(exitCode ?? 0);
+  } catch (error: any) {
+    const code = error.exitCode ?? 1;
+    hooks?.onExit?.(code);
+    throw new Error(`Command failed with exit code ${code}: ${error.message}`);
+  }
+};
+
+/**
+ * Calculates the total size of a directory recursively.
+ */
+export async function getFolderSize(dirPath: string): Promise<number> {
+  try {
+    const files = await readdir(dirPath, { withFileTypes: true });
+    const ArrayOfPromises = files.map(async (file) => {
+      const path = join(dirPath, file.name);
+      if (file.isDirectory()) {
+        try {
+          return await getFolderSize(path);
+        } catch {
+          return 0;
+        }
+      }
+      try {
+        const { size } = await stat(path);
+        return size;
+      } catch {
+        return 0;
+      }
+    });
+    const results = await Promise.all(ArrayOfPromises);
+    return results.reduce((acc, size) => acc + size, 0);
+  } catch {
+    return 0;
+  }
+}