@pipelab/core-node 1.0.1-beta.23

package/src/presets/test-c3-offline.ts

@@ -0,0 +1,78 @@
+import { PresetFn, SavedFile } from "@pipelab/shared";
+
+export const testC3Offline: PresetFn = async () => {
+  const packageWithElecton = "electron-package-node";
+  const steamUpload = "steam-upload-node";
+
+  const data: SavedFile = {
+    version: "3.0.0",
+    variables: [],
+    name: "C3 test without export",
+    description: "C3 test without export",
+    canvas: {
+      triggers: [
+        {
+          type: "event",
+          origin: {
+            pluginId: "system",
+            nodeId: "manual",
+          },
+          uid: "manual-start",
+          params: {},
+        },
+      ],
+      blocks: [
+        {
+          uid: packageWithElecton,
+          type: "action",
+          origin: {
+            nodeId: "package-to-electron",
+            pluginId: "electron",
+          },
+          params: {
+            "input-folder": {
+              editor: "editor",
+              value: `/home/quentin/Documents/Cyn Assets/app`,
+            },
+            arch: undefined,
+            platform: undefined,
+          },
+        },
+        {
+          uid: steamUpload,
+          type: "action",
+          origin: {
+            nodeId: "steam-upload",
+            pluginId: "steam",
+          },
+          params: {
+            folder: {
+              editor: "editor",
+              value: `{{ steps['${packageWithElecton}']['outputs']['output'] }}`,
+            },
+            appId: {
+              editor: "editor",
+              value: "3047200",
+            },
+            depotId: {
+              editor: "editor",
+              value: "3047201",
+            },
+            sdk: {
+              editor: "editor",
+              value: "/home/quentin/Documents/steamworkssdk/sdk",
+            },
+            username: {
+              editor: "editor",
+              value: "armaldio",
+            },
+          },
+        },
+      ],
+    },
+  };
+
+  return {
+    data,
+  };
+};

package/src/presets/test-c3-unzip.ts

@@ -0,0 +1,124 @@
+import { PresetFn, SavedFile } from "@pipelab/shared";
+
+export const testC3Unzip: PresetFn = async () => {
+  const exportConstructProjectId = "export-construct-project";
+  const unzipFileId = "unzip-file-node";
+  const packageWithElecton = "electron-package-node";
+  const steamUpload = "steam-upload-node";
+
+  const data: SavedFile = {
+    version: "3.0.0",
+    name: "From Construct to Steam",
+    description: "Export from Construct, package with Electron, then upload to Steam",
+    variables: [],
+    canvas: {
+      triggers: [
+        {
+          type: "event",
+          origin: {
+            pluginId: "system",
+            nodeId: "manual",
+          },
+          uid: "manual-start",
+          params: {},
+        },
+      ],
+      blocks: [
+        {
+          uid: exportConstructProjectId,
+          type: "action",
+          origin: {
+            nodeId: "export-construct-project",
+            pluginId: "construct",
+          },
+          params: {
+            file: {
+              editor: "editor",
+              value: `'/home/armaldio/Téléchargements/test.c3p'`,
+            },
+            username: {
+              editor: "editor",
+              value: '"a"',
+            },
+            password: {
+              editor: "editor",
+              value: '"a"',
+            },
+            version: {
+              editor: "editor",
+              value: '"395"',
+            },
+            headless: {
+              editor: "editor",
+              value: false,
+            },
+          },
+        },
+        {
+          uid: unzipFileId,
+          type: "action",
+          origin: {
+            nodeId: "unzip-file-node",
+            pluginId: "filesystem",
+          },
+          params: {
+            file: {
+              editor: "editor",
+              value: `steps['${exportConstructProjectId}']['outputs']['folder']`,
+            },
+          },
+        },
+        {
+          uid: packageWithElecton,
+          type: "action",
+          origin: {
+            nodeId: "electron:package",
+            pluginId: "electron",
+          },
+          params: {
+            "input-folder": {
+              editor: "editor",
+              value: `steps['${unzipFileId}']['outputs']['output']`,
+            },
+            arch: undefined,
+            platform: undefined,
+          },
+        },
+        {
+          uid: steamUpload,
+          type: "action",
+          origin: {
+            nodeId: "steam-upload",
+            pluginId: "steam",
+          },
+          params: {
+            folder: {
+              editor: "editor",
+              value: `steps['${packageWithElecton}']['outputs']['output']`,
+            },
+            appId: {
+              editor: "editor",
+              value: "'3047200'",
+            },
+            depotId: {
+              editor: "editor",
+              value: "'3047201'",
+            },
+            sdk: {
+              editor: "editor",
+              value: "'/home/armaldio/Documents/steamworkssdk/sdk'",
+            },
+            username: {
+              editor: "editor",
+              value: "'armaldio'",
+            },
+          },
+        },
+      ],
+    },
+  };
+
+  return {
+    data,
+  };
+};

package/src/runner.ts

@@ -0,0 +1,107 @@
+import { executeGraphWithHistory } from "./utils";
+import { setupConfigFile } from "./config";
+import { isDev, PipelabContext } from "./context";
+import { readFile, access, writeFile, mkdir } from "node:fs/promises";
+import { resolve, isAbsolute, join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import { savedFileMigrator } from "@pipelab/shared";
+import type { AppConfig } from "@pipelab/shared";
+import { registerMigrationHandlers } from "./migrations";
+import { registerAllHandlers } from "./handlers/index";
+
+export interface RunOptions {
+  userData?: string;
+  variables?: string;
+  output?: string;
+}
+
+export async function runPipelineCommand(file: string, options: RunOptions, version: string) {
+  const pipelinePath = isAbsolute(file) ? file : resolve(process.cwd(), file);
+
+  try {
+    await access(pipelinePath);
+  } catch (e) {
+    throw new Error(`Pipeline file not found at ${pipelinePath}`);
+  }
+
+  const pipelineContent = await readFile(pipelinePath, "utf-8");
+  const pipeline = JSON.parse(pipelineContent);
+
+  let finalPipeline = pipeline;
+  if (pipeline.version) {
+    finalPipeline = await savedFileMigrator.migrate(pipeline);
+  }
+
+  const {
+    graph: rawGraph,
+    variables: pipelineVariables,
+    projectName,
+    projectPath,
+    pipelineId,
+    canvas,
+    name,
+  } = finalPipeline;
+
+  const graph = rawGraph || canvas?.blocks;
+
+  if (!graph) {
+    throw new Error("Pipeline does not contain a valid graph or canvas.blocks");
+  }
+
+  let vars = pipelineVariables || finalPipeline.variables || [];
+  if (options.variables) {
+    vars = JSON.parse(options.variables);
+  }
+
+  const effectiveProjectName = projectName || name || "CLI Run";
+  const effectiveProjectPath = projectPath || process.cwd();
+  const effectivePipelineId = pipelineId || "cli-run";
+
+  console.log(`Executing pipeline: ${effectiveProjectName} (${effectivePipelineId})`);
+
+  if (!options.userData) throw new Error("userDataPath is required for runPipelineCommand");
+  const context = new PipelabContext({
+    userDataPath: options.userData,
+  });
+
+  await registerAllHandlers({ version, context });
+  registerMigrationHandlers(context);
+
+  const settings = await setupConfigFile<AppConfig>("settings", { context });
+  const config = await settings.getConfig();
+  const cachePath = config?.cacheFolder || tmpdir();
+
+  const abortController = new AbortController();
+
+  const { result, buildId } = await executeGraphWithHistory({
+    graph,
+    variables: vars,
+    projectName: effectiveProjectName,
+    projectPath: effectiveProjectPath,
+    pipelineId: effectivePipelineId,
+    cachePath: cachePath,
+    onNodeEnter: (node) => console.log(`[ENTER] ${node.name} (${node.uid})`),
+    onNodeExit: (node) => console.log(`[EXIT] ${node.name} (${node.uid})`),
+    onLog: (data) => {
+      if (data.type === "log") {
+        console.log(`[LOG] ${data.data.message}`);
+      }
+    },
+    abortSignal: abortController.signal,
+    context,
+  });
+
+  console.log(`Pipeline execution finished. Build ID: ${buildId}`);
+
+  if (options.output) {
+    const outputPath = isAbsolute(options.output)
+      ? options.output
+      : resolve(process.cwd(), options.output);
+
+    await mkdir(dirname(outputPath), { recursive: true });
+    await writeFile(outputPath, JSON.stringify(result, null, 2), "utf-8");
+    console.log(`Result saved to ${outputPath}`);
+  }
+
+  return result;
+}

package/src/server.ts

@@ -0,0 +1,80 @@
+import {
+  ensurePNPM,
+  PipelabContext,
+  isDev,
+  fetchPipelabAsset,
+  registerAllHandlers,
+  webSocketServer,
+} from "./index";
+import { getUiDevServerMissingWarning, uiDevPort } from "@pipelab/constants";
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import http from "http";
+import handler from "serve-handler";
+import { registerMigrationHandlers } from "./migrations";
+
+export interface ServeOptions {
+  port: string | number;
+  userData?: string;
+  nodePath?: string;
+  pnpmPath?: string;
+}
+
+export async function serveCommand(options: ServeOptions, version: string, _dirname: string) {
+  if (!options.userData) throw new Error("userDataPath is required for serveCommand");
+  const context = new PipelabContext({
+    userDataPath: options.userData,
+  });
+
+  let rawAssetFolder: string | undefined;
+  if (!isDev) {
+    rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", "latest", { context });
+  }
+
+  const server = http.createServer(async (request, response) => {
+    if (isDev) {
+      response.writeHead(200, { "Content-Type": "text/html" });
+      response.end(`
+        <html>
+          <body style="font-family: sans-serif; display: flex; flex-direction: column; align-items: center; justify-content: center; height: 100vh; margin: 0; background: #0f172a; color: #f8fafc;">
+            <h1 style="color: #38bdf8;">Pipelab Dev Mode</h1>
+            <p>The CLI server is running (API/WebSocket), but the UI is not served here in development.</p>
+            <p>Please open the UI through its own dev server (usually <a href="http://localhost:5173" style="color: #38bdf8;">http://localhost:5173</a>).</p>
+          </body>
+        </html>
+      `);
+      return;
+    }
+
+    if (!rawAssetFolder || !existsSync(rawAssetFolder)) {
+      response.writeHead(404, { "Content-Type": "text/plain" });
+      response.end(
+        `Error: UI directory not found at ${rawAssetFolder}.\n` +
+          "Please run 'pnpm build' in apps/ui to generate the distribution.",
+      );
+      return;
+    }
+
+    return handler(request, response, {
+      public: rawAssetFolder,
+    });
+  });
+
+  console.log(`Starting Pipelab server on port ${options.port}...`);
+  if (isDev) {
+    console.info(getUiDevServerMissingWarning());
+    console.log(`UI available at http://localhost:${uiDevPort}`);
+  } else {
+    console.log(`UI available at http://localhost:${options.port}`);
+  }
+
+  await registerAllHandlers({
+    version,
+    context,
+  });
+  registerMigrationHandlers(context);
+
+  await webSocketServer.start(Number(options.port), server);
+  return server;
+}

package/src/types/runner.ts

@@ -0,0 +1,101 @@
+import type { BrowserWindow } from "electron";
+import type { PipelabContext } from "../context";
+import type {
+  Action,
+  Condition,
+  Loop,
+  Expression,
+  Event,
+  SetOutputActionFn,
+  SetOutputLoopFn,
+  SetOutputExpressionFn,
+  ExtractInputsFromAction,
+  ExtractInputsFromCondition,
+  ExtractInputsFromLoop,
+  ExtractInputsFromEvent,
+  ExtractInputsFromExpression,
+} from "@pipelab/shared";
+
+export type {
+  Action,
+  Condition,
+  Loop,
+  Expression,
+  Event,
+  SetOutputActionFn,
+  SetOutputLoopFn,
+  SetOutputExpressionFn,
+  ExtractInputsFromAction,
+  ExtractInputsFromCondition,
+  ExtractInputsFromLoop,
+  ExtractInputsFromEvent,
+  ExtractInputsFromExpression,
+};
+
+export type RunnerCallbackFnArgument = {
+  done: () => void;
+  id: string;
+  log: (...args: Parameters<(typeof console)["log"]>) => void;
+};
+
+export type ActionRunnerData<ACTION extends Action> = {
+  log: typeof console.log;
+  setOutput: SetOutputActionFn<ACTION>;
+  inputs: ExtractInputsFromAction<ACTION>;
+  setMeta: (callback: (data: ACTION["meta"]) => ACTION["meta"]) => void;
+  meta: ACTION["meta"];
+  cwd: string;
+  paths: {
+    cache: string;
+    pnpm: string;
+    node: string;
+    userData: string;
+    modules: string;
+    thirdparty: string;
+  };
+  browserWindow: BrowserWindow;
+  abortSignal: AbortSignal;
+  context: PipelabContext;
+};
+
+export type ActionRunner<ACTION extends Action> = (data: ActionRunnerData<ACTION>) => Promise<void>;
+
+export type ConditionRunner<CONDITION extends Condition> = (data: {
+  log: typeof console.log;
+  inputs: ExtractInputsFromCondition<CONDITION>;
+  setMeta: (callback: (data: CONDITION["meta"]) => CONDITION["meta"]) => void;
+  meta: CONDITION["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<boolean>;
+
+export type LoopRunner<LOOP extends Loop> = (data: {
+  log: typeof console.log;
+  setOutput: SetOutputLoopFn<LOOP>;
+  inputs: ExtractInputsFromLoop<LOOP>;
+  setMeta: (callback: (data: LOOP["meta"]) => LOOP["meta"]) => void;
+  meta: LOOP["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<"step" | "exit">;
+
+export type ExpressionRunner<EXPRESSION extends Expression> = (data: {
+  log: typeof console.log;
+  setOutput: SetOutputExpressionFn<EXPRESSION>;
+  inputs: ExtractInputsFromExpression<EXPRESSION>;
+  setMeta: (callback: (data: EXPRESSION["meta"]) => EXPRESSION["meta"]) => void;
+  meta: EXPRESSION["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<string>;
+
+export type EventRunner<EVENT extends Event> = (data: {
+  log: typeof console.log;
+  inputs: ExtractInputsFromEvent<EVENT>;
+  setMeta: (callback: (data: EVENT["meta"]) => EVENT["meta"]) => void;
+  meta: EVENT["meta"];
+  cwd: string;
+  context: PipelabContext;
+}) => Promise<void>;
+
+export type Runner = ActionRunner<any> | LoopRunner<any> | EventRunner<any> | ConditionRunner<any>;

package/src/utils/fs-extras.ts

@@ -0,0 +1,182 @@
+import { mkdir, createWriteStream, createReadStream } from "node:fs";
+import { execa, Options, Subprocess } from "execa";
+import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod } from "node:fs/promises";
+import { join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import tar from "tar";
+import yauzl from "yauzl";
+import archiver from "archiver";
+import { pipeline } from "node:stream/promises";
+
+/**
+ * Ensures a directory exists and a file is created with default content if missing.
+ */
+export const ensure = async (filesPath: string, defaultContent = "{}") => {
+  await mkdirP(dirname(filesPath), { recursive: true });
+  try {
+    await access(filesPath);
+  } catch {
+    await writeFile(filesPath, defaultContent);
+  }
+};
+
+/**
+ * Generates a unique temporary folder.
+ */
+export const generateTempFolder = async (base?: string) => {
+  const targetBase = base || tmpdir();
+  await mkdirP(targetBase, { recursive: true });
+  const realPath = await realpath(targetBase);
+  const tempFolder = await mkdtemp(join(realPath, "pipelab-"));
+  return tempFolder;
+};
+
+/**
+ * Extracts a .tar.gz archive.
+ */
+export async function extractTarGz(archivePath: string, destinationDir: string): Promise<void> {
+  await mkdirP(destinationDir, { recursive: true });
+  await tar.x({
+    file: archivePath,
+    cwd: destinationDir,
+  });
+}
+
+/**
+ * Extracts a .zip archive.
+ */
+export async function extractZip(archivePath: string, destinationDir: string): Promise<void> {
+  await mkdirP(destinationDir, { recursive: true });
+  return new Promise((resolve, reject) => {
+    yauzl.open(archivePath, { lazyEntries: true }, (err, zipfile) => {
+      if (err || !zipfile) return reject(err || new Error("Could not open zip file"));
+      zipfile.on("error", reject);
+      zipfile.readEntry();
+      zipfile.on("entry", (entry) => {
+        const entryPath = join(destinationDir, entry.fileName);
+        if (/\/$/.test(entry.fileName)) {
+          mkdirP(entryPath, { recursive: true })
+            .then(() => zipfile.readEntry())
+            .catch(reject);
+        } else {
+          mkdirP(dirname(entryPath), { recursive: true })
+            .then(() => {
+              zipfile.openReadStream(entry, (err, readStream) => {
+                if (err || !readStream)
+                  return reject(err || new Error("Could not open read stream"));
+                readStream.on("error", reject);
+                const writeStream = createWriteStream(entryPath);
+                writeStream.on("error", reject);
+                writeStream.on("close", () => zipfile.readEntry());
+                readStream.pipe(writeStream);
+              });
+            })
+            .catch(reject);
+        }
+      });
+      zipfile.on("end", () => resolve());
+    });
+  });
+}
+
+/**
+ * Zips a folder.
+ */
+export const zipFolder = async (
+  from: string,
+  to: string,
+  log: typeof console.log = console.log,
+) => {
+  const output = createWriteStream(to);
+  const archive = archiver("zip", { zlib: { level: 9 } });
+  return new Promise<string>((resolve, reject) => {
+    output.on("close", () => {
+      log(archive.pointer() + " total bytes");
+      resolve(to);
+    });
+    archive.on("error", reject);
+    archive.pipe(output);
+    archive.directory(from, false);
+    archive.finalize();
+  });
+};
+
+/**
+ * Downloads a file with progress tracking.
+ */
+export interface DownloadHooks {
+  onProgress?: (data: { progress: number; downloadedSize: number }) => void;
+}
+
+export const downloadFile = async (
+  url: string,
+  localPath: string,
+  hooks?: DownloadHooks,
+  abortSignal?: AbortSignal,
+): Promise<void> => {
+  const response = await fetch(url, { signal: abortSignal });
+  if (!response.ok) throw new Error(`Failed to fetch file: ${response.statusText}`);
+  const contentLength = response.headers.get("content-length");
+  if (!contentLength) throw new Error("Content-Length header is missing");
+  const totalSize = parseInt(contentLength, 10);
+  let downloadedSize = 0;
+  const fileStream = createWriteStream(localPath);
+  const progressStream = new TransformStream({
+    transform(chunk, controller) {
+      downloadedSize += chunk.length;
+      const progress = (downloadedSize / totalSize) * 100;
+      hooks?.onProgress?.({ progress, downloadedSize });
+      controller.enqueue(chunk);
+    },
+  });
+  const readable = response.body?.pipeThrough(progressStream);
+  if (!readable) throw new Error("Failed to create a readable stream");
+  await pipeline(readable, fileStream);
+};
+
+export const runWithLiveLogs = async (
+  command: string,
+  args: string[],
+  execaOptions: Options,
+  log: typeof console.log,
+  hooks?: {
+    onStdout?: (data: string, subprocess: Subprocess) => void;
+    onStderr?: (data: string, subprocess: Subprocess) => void;
+    onExit?: (code: number) => void;
+    onCreated?: (subprocess: Subprocess) => void;
+  },
+  abortSignal?: AbortSignal,
+): Promise<void> => {
+  const subprocess = execa(command, args, {
+    ...execaOptions,
+    stdout: "pipe",
+    stderr: "pipe",
+    stdin: "pipe",
+    env: {
+      ...process.env,
+      ...execaOptions.env,
+      TERM: "xterm-256color",
+      FORCE_STDERR_LOGGING: "1",
+    },
+    cancelSignal: abortSignal,
+  });
+
+  hooks?.onCreated?.(subprocess);
+
+  subprocess.stdout?.on("data", (data: Buffer) => {
+    hooks?.onStdout?.(data.toString(), subprocess);
+  });
+
+  subprocess.stderr?.on("data", (data: Buffer) => {
+    hooks?.onStderr?.(data.toString(), subprocess);
+  });
+
+  try {
+    const { exitCode } = await subprocess;
+    hooks?.onExit?.(exitCode ?? 0);
+  } catch (error: any) {
+    const code = error.exitCode ?? 1;
+    hooks?.onExit?.(code);
+    throw new Error(`Command failed with exit code ${code}: ${error.message}`);
+  }
+};