@pipelab/core-node 1.0.0-beta.0 → 1.0.0-beta.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipelab/core-node",
-  "version": "1.0.0-beta.0",
+  "version": "1.0.0-beta.10",
   "private": false,
   "description": "The Pipelab automation engine for Node.js",
   "license": "FSL-1.1-MIT",
@@ -26,6 +26,7 @@
     "access": "public"
   },
   "dependencies": {
+    "@supabase/supabase-js": "2.99.3",
     "adm-zip": "0.5.16",
     "archiver": "7.0.1",
     "check-disk-space": "3.4.0",
@@ -39,9 +40,8 @@
     "type-fest": "4.39.0",
     "ws": "8.18.3",
     "yauzl": "2.10.0",
-    "@supabase/supabase-js": "2.99.3",
-    "@pipelab/constants": "1.0.0-beta.0",
-    "@pipelab/shared": "1.0.0-beta.0"
+    "@pipelab/constants": "1.0.0-beta.8",
+    "@pipelab/shared": "1.0.0-beta.6"
   },
   "devDependencies": {
     "@types/adm-zip": "0.5.7",
@@ -54,7 +54,7 @@
     "@types/yauzl": "2.10.3",
     "tsdown": "0.21.2",
     "typescript": "^5.0.0",
-    "@pipelab/tsconfig": "1.0.0-beta.0"
+    "@pipelab/tsconfig": "1.0.0-beta.6"
   },
   "scripts": {
     "build": "tsdown",

package/scratch/simulate-updates.ts

@@ -0,0 +1,118 @@
+import pacote from "pacote";
+import semver from "semver";
+import { fetchLatestDesktopRelease } from "../src/utils/github.js";
+
+const PACKAGES = [
+  // Apps & Core
+  "@pipelab/cli",
+  "@pipelab/ui",
+  "@pipelab/shared",
+  "@pipelab/core-node",
+  "@pipelab/constants",
+  "@pipelab/migration",
+
+  // Plugins
+  "@pipelab/plugin-construct",
+  "@pipelab/plugin-core",
+  "@pipelab/plugin-discord",
+  "@pipelab/plugin-electron",
+  "@pipelab/plugin-filesystem",
+  "@pipelab/plugin-itch",
+  "@pipelab/plugin-minify",
+  "@pipelab/plugin-netlify",
+  "@pipelab/plugin-nvpatch",
+  "@pipelab/plugin-poki",
+  "@pipelab/plugin-steam",
+  "@pipelab/plugin-system",
+  "@pipelab/plugin-tauri",
+
+  // Assets
+  "@pipelab/asset-discord",
+  "@pipelab/asset-electron",
+  "@pipelab/asset-netlify",
+  "@pipelab/asset-tauri"
+];
+
+async function simulate() {
+  console.log("==========================================");
+  console.log("     PIPELAB UPDATE FLOW SIMULATOR        ");
+  console.log("==========================================\n");
+
+  // 1. Simulate Desktop GitHub Release Update
+  console.log("--- 1. Simulating Desktop Application (GitHub Releases) ---");
+  try {
+    const stableApp = await fetchLatestDesktopRelease({ allowPrerelease: false });
+    const betaApp = await fetchLatestDesktopRelease({ allowPrerelease: true });
+
+    console.log(`[STABLE] Resolved version: ${stableApp ? stableApp.tag_name : "None (Stable users protected)"}`);
+    console.log(`[BETA]   Resolved version: ${betaApp ? betaApp.tag_name : "None"}`);
+  } catch (error) {
+    console.error("Failed to fetch desktop releases from GitHub:", error);
+  }
+
+  console.log("\n--- 2. Simulating NPM Packages (CLI, UI, Plugins, Assets) ---");
+  console.log("Fetching data from NPM registry. Please wait...\n");
+
+  const results: Array<{
+    package: string;
+    stable: string;
+    beta: string;
+    notes: string;
+  }> = [];
+
+  for (const pkg of PACKAGES) {
+    try {
+      const packument = await pacote.packument(pkg);
+      const stableVersion = packument["dist-tags"]?.latest || "N/A";
+      const betaVersion = packument["dist-tags"]?.beta || "N/A";
+
+      let notes = "";
+      if (stableVersion === "N/A" && betaVersion === "N/A") {
+        notes = "⚠️ Package not published on NPM";
+      } else if (betaVersion !== "N/A" && stableVersion !== "N/A") {
+        if (betaVersion === stableVersion) {
+          notes = "Stable and Beta are in sync";
+        } else {
+          // Attempt semver comparison
+          const cleanStable = semver.coerce(stableVersion);
+          const cleanBeta = semver.coerce(betaVersion);
+          if (cleanStable && cleanBeta) {
+            const comp = semver.compare(stableVersion, betaVersion);
+            if (comp < 0) {
+              notes = "🚀 Beta is ahead";
+            } else if (comp > 0) {
+              notes = "Stable is ahead";
+            } else {
+              notes = "Stable and Beta are in sync (coerced)";
+            }
+          } else {
+            notes = "Different versions (non-semver)";
+          }
+        }
+      } else if (betaVersion !== "N/A" && stableVersion === "N/A") {
+        notes = "Only Beta version exists";
+      } else if (stableVersion !== "N/A" && betaVersion === "N/A") {
+        notes = "No Beta tag published";
+      }
+
+      results.push({
+        package: pkg,
+        stable: stableVersion,
+        beta: betaVersion,
+        notes
+      });
+    } catch (error: any) {
+      results.push({
+        package: pkg,
+        stable: "ERROR",
+        beta: "ERROR",
+        notes: `Failed to fetch: ${error.message}`
+      });
+    }
+  }
+
+  console.table(results);
+  console.log("\n==========================================\n");
+}
+
+simulate().catch(console.error);

package/src/config.ts

@@ -66,6 +66,16 @@ export const setupConfigFile = async <T>(
       try {
         json = await migrator.migrate(originalJson, {
           debug: false,
+          onStep: async (state: any, version: string) => {
+            const parsedPath = path.parse(filesPath);
+            const versionedPath = path.join(parsedPath.dir, `${parsedPath.name}.v${version}.json`);
+            try {
+              await fs.writeFile(versionedPath, JSON.stringify(state));
+              logger().info(`Intermediate backup created for ${name} at ${versionedPath}`);
+            } catch (e) {
+              logger().error(`Failed to create intermediate backup for ${name} at v${version}:`, e);
+            }
+          },
         });
       } catch (e) {
         logger().error(`Error migrating config ${name}:`, e);
@@ -75,21 +85,6 @@ export const setupConfigFile = async <T>(
       const originalVersion = originalJson?.version;
       const newVersion = json?.version;
 
-      // Check if migration actually changed the version
-      if (originalVersion !== newVersion && content !== undefined) {
-        // Backup previous file before overwriting
-        const parsedPath = path.parse(filesPath);
-        const versionSuffix = originalVersion || "unknown";
-        const backupPath = path.join(parsedPath.dir, `${parsedPath.name}.${versionSuffix}.bak`);
-
-        try {
-          await fs.copyFile(filesPath, backupPath);
-          logger().info(`Backup created for ${name} at ${backupPath} (version ${versionSuffix})`);
-        } catch (e) {
-          logger().error(`Failed to create backup for ${name}:`, e);
-        }
-      }
-
       // Save back migrated config if changed or if it's a new file
       if (originalVersion !== newVersion || content === undefined) {
         try {

package/src/context.ts

@@ -12,7 +12,7 @@ const _dirname =
 
 export const isDev = process.env.NODE_ENV === "development";
 
-export const getDefaultUserDataPath = () => {
+export const getDefaultUserDataPath = (env?: "dev" | "beta" | "prod") => {
   const base = (() => {
     switch (platform()) {
       case "win32":
@@ -24,7 +24,10 @@ export const getDefaultUserDataPath = () => {
     }
   })();
 
-  return join(base, "@pipelab", isDev ? "app-dev" : "app");
+  const mode = env ?? (isDev ? "dev" : "prod");
+  const folder = mode === "dev" ? "app-dev" : mode === "beta" ? "app-beta" : "app";
+
+  return join(base, "@pipelab", folder);
 };
 
 /**
@@ -45,13 +48,16 @@ export const projectRoot = findProjectRoot(_dirname);
 
 export interface PipelabContextOptions {
   userDataPath: string;
+  releaseTag?: string;
 }
 
 export class PipelabContext {
   public readonly userDataPath: string;
+  public readonly releaseTag: string;
 
   constructor(options: PipelabContextOptions) {
     this.userDataPath = options.userDataPath;
+    this.releaseTag = options.releaseTag || "latest";
   }
 
   getPackagesPath(...subpaths: string[]) {

package/src/handler-func.ts

@@ -53,9 +53,9 @@ export const handleConditionExecute = async (
     .find((plugin) => plugin.id === pluginId)
     ?.nodes.find((node: any) => node.node.id === nodeId) as
     | {
-      node: Condition;
-      runner: ConditionRunner<any>;
-    }
+        node: Condition;
+        runner: ConditionRunner<any>;
+      }
     | undefined;
 
   if (!node) {
@@ -136,9 +136,9 @@ export const handleActionExecute = async (
     .find((plugin) => plugin.id === pluginId)
     ?.nodes.find((node: any) => node.node.id === nodeId) as
     | {
-      node: Action;
-      runner: ActionRunner<any>;
-    }
+        node: Action;
+        runner: ActionRunner<any>;
+      }
     | undefined;
 
   if (!node) {

package/src/handlers/build-history.ts

@@ -93,17 +93,13 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
 
         // 2. Filter by maxEntries (sort by date first to keep the newest)
         if (maxEntries > 0 && entries.length > maxEntries) {
-          entries = entries
-            .sort((a, b) => b.createdAt - a.createdAt)
-            .slice(0, maxEntries);
+          entries = entries.sort((a, b) => b.createdAt - a.createdAt).slice(0, maxEntries);
         }
 
         if (entries.length < originalCount) {
           this.logger
             .logger()
-            .info(
-              `[${pipelineId}] Pruned ${originalCount - entries.length} history entries.`,
-            );
+            .info(`[${pipelineId}] Pruned ${originalCount - entries.length} history entries.`);
           await this.savePipelineHistory(pipelineId, entries);
         }
       }
@@ -271,8 +267,10 @@ export class BuildHistoryStorage implements IBuildHistoryStorage {
       await unlink(pipelinePath);
       this.logger.logger().info(`Cleared history for pipeline "${pipelineId}"`);
     } catch (error: any) {
-      if (error.code === 'ENOENT') {
-        this.logger.logger().warn(`No history file found for pipeline "${pipelineId}". Nothing to clear.`);
+      if (error.code === "ENOENT") {
+        this.logger
+          .logger()
+          .warn(`No history file found for pipeline "${pipelineId}". Nothing to clear.`);
         return;
       }
       this.logger.logger().error(`Failed to clear history for pipeline "${pipelineId}":`, error);

package/src/handlers/history.ts

@@ -127,8 +127,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to get build history entries",
+          ipcError: error instanceof Error ? error.message : "Failed to get build history entries",
         },
       });
     }
@@ -158,8 +157,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to update build history entry",
+          ipcError: error instanceof Error ? error.message : "Failed to update build history entry",
         },
       });
     }
@@ -189,8 +187,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError:
-            error instanceof Error ? error.message : "Failed to delete build history entry",
+          ipcError: error instanceof Error ? error.message : "Failed to delete build history entry",
         },
       });
     }
@@ -250,7 +247,8 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
         type: "end",
         data: {
           type: "error",
-          ipcError: error instanceof Error ? error.message : "Failed to clear build history for pipeline",
+          ipcError:
+            error instanceof Error ? error.message : "Failed to clear build history for pipeline",
         },
       });
     }
@@ -290,7 +288,7 @@ export const registerHistoryHandlers = (context: PipelabContext) => {
   handle("build-history:configure", async (_, { send, value }) => {
     try {
       logger().info("Updating build history configuration:", value.config);
-      
+
       const settings = await setupConfigFile<AppConfig>("settings", { context });
       const currentConfig = await settings.getConfig();
 

package/src/handlers/index.ts

@@ -26,10 +26,10 @@ export const registerAllHandlers = async (options: {
   registerSystemHandlers(options);
 
   const { registerPlugins } = usePlugins();
-  const plugins = await builtInPlugins({
+  // Execute in the background! The plugins will be dynamically registered and broadcasted to the UI.
+  builtInPlugins({
     context,
   });
-  registerPlugins(plugins as any);
 };
 
 export { registerShellHandlers } from "./shell";

package/src/index.ts

@@ -14,3 +14,4 @@ export * from "./runner";
 export * from "./migrations";
 export * from "./server";
 export * from "./utils";
+export * from "./utils/github";

package/src/plugins-registry.ts

@@ -23,9 +23,9 @@ const DEFAULT_PLUGIN_IDS = [
 export const loadPipelabPlugin = async (id: string, options: { context: PipelabContext }) => {
   try {
     const packageName = `@pipelab/plugin-${id}`;
-    const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, "latest", {
+    const { packageDir, entryPoint } = await fetchPipelabPlugin(packageName, options.context.releaseTag, {
       context: options.context,
-      installDeps: true,
+      installDeps: false,
     });
 
     console.log(`[Plugins] [${id}] Attempting to import from: ${entryPoint}`);
@@ -34,7 +34,7 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
       try {
         const files = await readdir(packageDir, { recursive: true });
         console.log(`[Plugins] [${id}] Directory contents:`, files);
-      } catch (e) { }
+      } catch (e) {}
     }
 
     const pluginModule = await import(pathToFileURL(entryPoint).href);
@@ -52,27 +52,32 @@ export const loadPipelabPlugin = async (id: string, options: { context: PipelabC
 };
 
 export const builtInPlugins = async (options: { context: PipelabContext }) => {
-  console.log("[Plugins] Finalizing default plugins list...");
+  console.log("[Plugins] Starting background plugin loading...");
 
   // Pre-ensure Node.js and PNPM once in parallel so plugins don't have to wait for them
   sendStartupProgress("Preparing environment...");
-  await Promise.all([
-    ensureNodeJS(options.context),
-    ensurePNPM(options.context),
-  ]);
+  await Promise.all([ensureNodeJS(options.context), ensurePNPM(options.context)]);
 
-  const results = await Promise.allSettled(
-    DEFAULT_PLUGIN_IDS.map(async (id) => {
-      sendStartupProgress(`Loading plugin: ${id}`);
-      return loadPipelabPlugin(id, options);
-    }),
-  );
+  const { usePlugins } = await import("@pipelab/shared");
+  const { registerPlugins } = usePlugins();
+  const { webSocketServer } = await import("./index");
 
-  const plugins = results
-    .filter((r) => r.status === "fulfilled")
-    .map((r) => (r as PromiseFulfilledResult<any>).value);
+  // Load plugins asynchronously in the background
+  (async () => {
+    for (const id of DEFAULT_PLUGIN_IDS) {
+      sendStartupProgress(`Loading plugin: ${id}`);
+      const plugin = await loadPipelabPlugin(id, options);
+      if (plugin) {
+        registerPlugins([plugin]);
+        webSocketServer.broadcast("plugin:loaded", { plugin });
+      }
+    }
+    console.log("[Plugins] All default plugins loaded.");
+    sendStartupProgress("All plugins loaded.");
+    setTimeout(() => {
+      webSocketServer.broadcast("startup:progress", { type: "ready" });
+    }, 2000);
+  })();
 
-  const filtered = plugins.filter(Boolean).flat();
-  console.log(`[Plugins] Successfully loaded ${filtered.length} default plugins`);
-  return filtered;
+  return [];
 };

package/src/runner.ts

@@ -79,10 +79,7 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
   const cloudRunId = process.env.CLOUD_RUN_ID;
   if (options.cloud && cloudRunId) {
     const { createClient } = await import("@supabase/supabase-js");
-    supabase = createClient(
-      process.env.SUPABASE_URL!,
-      process.env.SUPABASE_SERVICE_ROLE_KEY!
-    );
+    supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!);
     console.log(`Cloud mode enabled. Streaming logs for run: ${cloudRunId}`);
   }
 
@@ -97,23 +94,29 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
       onNodeEnter: (node) => {
         console.log(`[ENTER] ${node.name} (${node.uid})`);
         if (supabase) {
-          supabase.from("cloud_run_logs").insert({
-            run_id: cloudRunId,
-            message: `[ENTER] ${node.name} (${node.uid})`,
-            node_uid: node.uid,
-            type: "node-enter"
-          }).then();
+          supabase
+            .from("cloud_run_logs")
+            .insert({
+              run_id: cloudRunId,
+              message: `[ENTER] ${node.name} (${node.uid})`,
+              node_uid: node.uid,
+              type: "node-enter",
+            })
+            .then();
         }
       },
       onNodeExit: (node) => {
         console.log(`[EXIT] ${node.name} (${node.uid})`);
         if (supabase) {
-          supabase.from("cloud_run_logs").insert({
-            run_id: cloudRunId,
-            message: `[EXIT] ${node.name} (${node.uid})`,
-            node_uid: node.uid,
-            type: "node-exit"
-          }).then();
+          supabase
+            .from("cloud_run_logs")
+            .insert({
+              run_id: cloudRunId,
+              message: `[EXIT] ${node.name} (${node.uid})`,
+              node_uid: node.uid,
+              type: "node-exit",
+            })
+            .then();
         }
       },
       onLog: (data, node) => {
@@ -121,12 +124,15 @@ export async function runPipelineCommand(file: string, options: RunOptions, vers
           const message = data.data.message.join(" ");
           console.log(`[LOG] ${message}`);
           if (supabase) {
-            supabase.from("cloud_run_logs").insert({
-              run_id: cloudRunId,
-              message: message,
-              node_uid: node?.uid,
-              type: "log"
-            }).then();
+            supabase
+              .from("cloud_run_logs")
+              .insert({
+                run_id: cloudRunId,
+                message: message,
+                node_uid: node?.uid,
+                type: "log",
+              })
+              .then();
           }
         }
       },

package/src/server.ts

@@ -38,13 +38,15 @@ export const sendStartupReady = () => {
 
 export async function serveCommand(options: ServeOptions, version: string, _dirname: string) {
   if (!options.userData) throw new Error("userDataPath is required for serveCommand");
+  const releaseTag = version.includes("beta") ? "beta" : "latest";
   const context = new PipelabContext({
     userDataPath: options.userData,
+    releaseTag,
   });
 
   let rawAssetFolder: string | undefined;
   if (!isDev) {
-    rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", "latest", { context });
+    rawAssetFolder = await fetchPipelabAsset("@pipelab/ui", releaseTag, { context });
   }
 
   const server = http.createServer(async (request, response) => {
@@ -66,7 +68,7 @@ export async function serveCommand(options: ServeOptions, version: string, _dirn
       response.writeHead(404, { "Content-Type": "text/plain" });
       response.end(
         `Error: UI directory not found at ${rawAssetFolder}.\n` +
-        "Please run 'pnpm build' in apps/ui to generate the distribution.",
+          "Please run 'pnpm build' in apps/ui to generate the distribution.",
       );
       return;
     }

package/src/utils/fs-extras.ts

@@ -1,6 +1,15 @@
 import { mkdir, createWriteStream, createReadStream } from "node:fs";
 import { execa, Options, Subprocess } from "execa";
-import { mkdir as mkdirP, access, writeFile, realpath, mkdtemp, chmod, stat, readdir } from "node:fs/promises";
+import {
+  mkdir as mkdirP,
+  access,
+  writeFile,
+  realpath,
+  mkdtemp,
+  chmod,
+  stat,
+  readdir,
+} from "node:fs/promises";
 import { join, dirname } from "node:path";
 import { tmpdir } from "node:os";
 import tar from "tar";

package/src/utils/github.ts

@@ -0,0 +1,119 @@
+import semver from "semver";
+
+export interface GitHubRelease {
+  tag_name: string;
+  prerelease: boolean;
+  published_at: string;
+  html_url: string;
+  assets: Array<{
+    name: string;
+    browser_download_url: string;
+  }>;
+}
+
+export interface FetchReleaseOptions {
+  repo?: string;
+  allowPrerelease?: boolean;
+}
+
+/**
+ * Fetches all releases for a specific package from GitHub.
+ * Tags are expected to follow the pattern "{packageName}@{version}"
+ */
+export async function fetchPackageReleases(
+  packageName: string,
+  options: FetchReleaseOptions = {},
+): Promise<GitHubRelease[]> {
+  const { repo = "CynToolkit/pipelab", allowPrerelease = false } = options;
+  const url = `https://api.github.com/repos/${repo}/releases`;
+
+  console.log(`[GitHub] Fetching releases for ${packageName} from ${url}...`);
+
+  try {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Pipelab-Desktop-Updater",
+        Accept: "application/vnd.github.v3+json",
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
+    }
+
+    const releases: GitHubRelease[] = await response.json();
+
+    // Filter for releases that follow the {packageName}@X.Y.Z tag pattern
+    const packageReleases = releases
+      .filter((r) => r.tag_name.startsWith(`${packageName}@`))
+      .filter((r) => allowPrerelease || !r.prerelease);
+
+    return packageReleases;
+  } catch (error) {
+    console.error(`[GitHub] Failed to fetch releases for ${packageName}:`, error);
+    return [];
+  }
+}
+
+/**
+ * Fetches the latest release for a specific package.
+ * Supports PIPELAB_OVERRIDE_RELEASE environment variable.
+ */
+export async function fetchLatestPackageRelease(
+  packageName: string,
+  options: FetchReleaseOptions = {},
+): Promise<GitHubRelease | null> {
+  const releases = await fetchPackageReleases(packageName, options);
+
+  if (releases.length === 0) {
+    return null;
+  }
+
+  // Handle environment variable override
+  const override = process.env.PIPELAB_OVERRIDE_RELEASE;
+  if (override) {
+    const targetTag = override.includes("@") ? override : `${packageName}@${override}`;
+    const release = releases.find((r) => r.tag_name === targetTag);
+    if (release) {
+      console.log(
+        `[GitHub] Using release override from PIPELAB_OVERRIDE_RELEASE: ${release.tag_name}`,
+      );
+      return release;
+    }
+    console.warn(`[GitHub] Override release ${override} not found for package ${packageName}`);
+  }
+
+  // Sort by semver
+  const getVersion = (tagName: string) => tagName.split("@").pop() || "0.0.0";
+
+  releases.sort((a, b) => {
+    const vA = getVersion(a.tag_name);
+    const vB = getVersion(b.tag_name);
+    if (semver.valid(vA) && semver.valid(vB)) {
+      return semver.rcompare(vA, vB);
+    }
+    return new Date(b.published_at).getTime() - new Date(a.published_at).getTime();
+  });
+
+  return releases[0];
+}
+
+/**
+ * Fetches all releases from GitHub and finds the latest one that matches the Pipelab Desktop app tag.
+ * This ensures we don't accidentally pick a package release (e.g. @pipelab/shared) as the "latest".
+ */
+export async function fetchLatestDesktopRelease(
+  options: FetchReleaseOptions = {},
+): Promise<GitHubRelease | null> {
+  const latest = await fetchLatestPackageRelease("@pipelab/app", options);
+
+  if (latest) {
+    console.log(
+      `[GitHub] Found latest desktop release: ${latest.tag_name} (${latest.prerelease ? "pre-release" : "stable"})`,
+    );
+  } else {
+    console.warn(`[GitHub] No desktop releases found`);
+  }
+
+  return latest;
+}