@pipelab/core-node 1.0.0-beta.2 → 1.0.0-beta.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipelab/core-node",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.21",
   "private": false,
   "description": "The Pipelab automation engine for Node.js",
   "license": "FSL-1.1-MIT",
@@ -40,8 +40,8 @@
     "type-fest": "4.39.0",
     "ws": "8.18.3",
     "yauzl": "2.10.0",
-    "@pipelab/constants": "1.0.0-beta.1",
-    "@pipelab/shared": "1.0.0-beta.1"
+    "@pipelab/constants": "1.0.0-beta.19",
+    "@pipelab/shared": "1.0.0-beta.17"
   },
   "devDependencies": {
     "@types/adm-zip": "0.5.7",
@@ -52,13 +52,17 @@
     "@types/tar": "7.0.87",
     "@types/ws": "8.18.1",
     "@types/yauzl": "2.10.3",
+    "memfs": "4.57.6",
     "tsdown": "0.21.2",
     "typescript": "^5.0.0",
-    "@pipelab/tsconfig": "1.0.0-beta.1"
+    "vitest": "3.1.4",
+    "@pipelab/tsconfig": "1.0.0-beta.17"
   },
   "scripts": {
     "build": "tsdown",
     "format": "oxfmt .",
-    "lint": "oxlint ."
+    "lint": "oxlint .",
+    "typecheck": "tsc -b",
+    "test": "vitest run"
   }
 }

package/scratch/simulate-updates.ts

@@ -0,0 +1,120 @@
+import pacote from "pacote";
+import semver from "semver";
+import { fetchLatestDesktopRelease } from "../src/utils/github.js";
+
+const PACKAGES = [
+  // Apps & Core
+  "@pipelab/cli",
+  "@pipelab/ui",
+  "@pipelab/shared",
+  "@pipelab/core-node",
+  "@pipelab/constants",
+  "@pipelab/migration",
+
+  // Plugins
+  "@pipelab/plugin-construct",
+  "@pipelab/plugin-core",
+  "@pipelab/plugin-discord",
+  "@pipelab/plugin-electron",
+  "@pipelab/plugin-filesystem",
+  "@pipelab/plugin-itch",
+  "@pipelab/plugin-minify",
+  "@pipelab/plugin-netlify",
+  "@pipelab/plugin-nvpatch",
+  "@pipelab/plugin-poki",
+  "@pipelab/plugin-steam",
+  "@pipelab/plugin-system",
+  "@pipelab/plugin-tauri",
+
+  // Assets
+  "@pipelab/asset-discord",
+  "@pipelab/asset-electron",
+  "@pipelab/asset-netlify",
+  "@pipelab/asset-tauri",
+];
+
+async function simulate() {
+  console.log("==========================================");
+  console.log("     PIPELAB UPDATE FLOW SIMULATOR        ");
+  console.log("==========================================\n");
+
+  // 1. Simulate Desktop GitHub Release Update
+  console.log("--- 1. Simulating Desktop Application (GitHub Releases) ---");
+  try {
+    const stableApp = await fetchLatestDesktopRelease({ allowPrerelease: false });
+    const betaApp = await fetchLatestDesktopRelease({ allowPrerelease: true });
+
+    console.log(
+      `[STABLE] Resolved version: ${stableApp ? stableApp.tag_name : "None (Stable users protected)"}`,
+    );
+    console.log(`[BETA]   Resolved version: ${betaApp ? betaApp.tag_name : "None"}`);
+  } catch (error) {
+    console.error("Failed to fetch desktop releases from GitHub:", error);
+  }
+
+  console.log("\n--- 2. Simulating NPM Packages (CLI, UI, Plugins, Assets) ---");
+  console.log("Fetching data from NPM registry. Please wait...\n");
+
+  const results: Array<{
+    package: string;
+    stable: string;
+    beta: string;
+    notes: string;
+  }> = [];
+
+  for (const pkg of PACKAGES) {
+    try {
+      const packument = await pacote.packument(pkg);
+      const stableVersion = packument["dist-tags"]?.latest || "N/A";
+      const betaVersion = packument["dist-tags"]?.beta || "N/A";
+
+      let notes = "";
+      if (stableVersion === "N/A" && betaVersion === "N/A") {
+        notes = "⚠️ Package not published on NPM";
+      } else if (betaVersion !== "N/A" && stableVersion !== "N/A") {
+        if (betaVersion === stableVersion) {
+          notes = "Stable and Beta are in sync";
+        } else {
+          // Attempt semver comparison
+          const cleanStable = semver.coerce(stableVersion);
+          const cleanBeta = semver.coerce(betaVersion);
+          if (cleanStable && cleanBeta) {
+            const comp = semver.compare(stableVersion, betaVersion);
+            if (comp < 0) {
+              notes = "🚀 Beta is ahead";
+            } else if (comp > 0) {
+              notes = "Stable is ahead";
+            } else {
+              notes = "Stable and Beta are in sync (coerced)";
+            }
+          } else {
+            notes = "Different versions (non-semver)";
+          }
+        }
+      } else if (betaVersion !== "N/A" && stableVersion === "N/A") {
+        notes = "Only Beta version exists";
+      } else if (stableVersion !== "N/A" && betaVersion === "N/A") {
+        notes = "No Beta tag published";
+      }
+
+      results.push({
+        package: pkg,
+        stable: stableVersion,
+        beta: betaVersion,
+        notes,
+      });
+    } catch (error: any) {
+      results.push({
+        package: pkg,
+        stable: "ERROR",
+        beta: "ERROR",
+        notes: `Failed to fetch: ${error.message}`,
+      });
+    }
+  }
+
+  console.table(results);
+  console.log("\n==========================================\n");
+}
+
+simulate().catch(console.error);

package/src/config.test.ts

@@ -0,0 +1,232 @@
+import { describe, test, expect, beforeEach, vi } from "vitest";
+import {
+  setupConfigFile,
+  setupProjectsConfigFile,
+  deletePipelineConfigFileByName,
+  deletePipelineConfigFileByPath,
+} from "./config";
+import { FileRepo } from "@pipelab/shared";
+import { PipelabContext } from "./context";
+import fs from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { vol } from "memfs";
+
+// Mock the node:fs and node:fs/promises modules to use memfs
+vi.mock("node:fs", async () => {
+  const memfs = await import("memfs");
+  return {
+    ...memfs.fs,
+    default: memfs.fs,
+  };
+});
+
+vi.mock("node:fs/promises", async () => {
+  const memfs = await import("memfs");
+  return {
+    ...memfs.fs.promises,
+    default: memfs.fs.promises,
+  };
+});
+
+describe("setupConfigFile & Backup Creation", () => {
+  let tempDir: string;
+  let context: PipelabContext;
+
+  beforeEach(() => {
+    vol.reset();
+    tempDir = "/tmp/pipelab-test-config";
+    context = new PipelabContext({ userDataPath: tempDir });
+  });
+
+  test("should migrate config and create backup files on disk", async () => {
+    // 1. Setup V1 configuration file in the context's config path
+    const configDir = context.getConfigPath();
+    await fs.mkdir(configDir, { recursive: true });
+
+    const projectsFilePath = context.getProjectsPath();
+
+    const v1Config = {
+      version: "1.0.0",
+      data: {
+        "pipeline-1": {
+          project: "main",
+          type: "internal",
+          configName: "pipeline-1",
+          lastModified: "2026-06-04",
+        },
+      },
+    };
+
+    await fs.writeFile(projectsFilePath, JSON.stringify(v1Config));
+
+    // 2. Initialize setupConfigFile
+    const configInstance = await setupProjectsConfigFile(context);
+
+    // Verify file exists
+    expect(existsSync(projectsFilePath)).toBe(true);
+
+    // 3. Retrieve config (this triggers the migration process and onStep callback)
+    const migratedConfig = await configInstance.getConfig();
+
+    // 4. Assert the main config is updated on disk to version 3.0.0
+    expect(migratedConfig.version).toBe("3.0.0");
+
+    const updatedContent = JSON.parse(await fs.readFile(projectsFilePath, "utf8"));
+    expect(updatedContent.version).toBe("3.0.0");
+
+    // 5. Assert that the intermediate backup file was created on disk
+    const backupFilePath = path.join(configDir, "projects.v2.0.0.json");
+    expect(existsSync(backupFilePath)).toBe(true);
+
+    const backupContent = JSON.parse(await fs.readFile(backupFilePath, "utf8"));
+    expect(backupContent.version).toBe("2.0.0");
+    expect(backupContent.projects).toHaveLength(1);
+    expect(backupContent.pipelines).toHaveLength(1);
+  });
+
+  test("should create default config file if missing on setup", async () => {
+    const configDir = context.getConfigPath();
+    const projectsFilePath = context.getProjectsPath();
+
+    expect(existsSync(projectsFilePath)).toBe(false);
+
+    // Initialize setupConfigFile
+    await setupProjectsConfigFile(context);
+
+    // Should create file with default value
+    expect(existsSync(projectsFilePath)).toBe(true);
+    const content = JSON.parse(await fs.readFile(projectsFilePath, "utf8"));
+    expect(content.version).toBe("3.0.0");
+    expect(content.projects).toHaveLength(1);
+    expect(content.pipelines).toEqual([]);
+  });
+
+  test("should fallback to default config and preserve corrupted file if parsing fails", async () => {
+    const configDir = context.getConfigPath();
+    await fs.mkdir(configDir, { recursive: true });
+    const projectsFilePath = context.getProjectsPath();
+
+    await fs.writeFile(projectsFilePath, "{ corrupted json... }");
+
+    const configInstance = await setupProjectsConfigFile(context);
+    const config = await configInstance.getConfig();
+
+    expect(config.version).toBe("3.0.0");
+    expect(config.projects).toHaveLength(1);
+    expect(config.pipelines).toEqual([]);
+
+    // Verify a timestamped corrupted backup file exists
+    const files = await fs.readdir(configDir);
+    const corruptedFile = files.find(
+      (f) => f.startsWith("projects.corrupted.") && f.endsWith(".json"),
+    );
+    expect(corruptedFile).toBeDefined();
+    const corruptedContent = await fs.readFile(path.join(configDir, corruptedFile!), "utf8");
+    expect(corruptedContent).toBe("{ corrupted json... }");
+  });
+
+  test("should fallback to default config and preserve file if migration throws", async () => {
+    const configDir = context.getConfigPath();
+    await fs.mkdir(configDir, { recursive: true });
+    const projectsFilePath = context.getProjectsPath();
+
+    const customMigrator = {
+      defaultValue: { version: "2.0.0", projects: [], pipelines: [] } as any,
+      migrate: async () => {
+        throw new Error("Migration failed!");
+      },
+    };
+
+    await fs.writeFile(projectsFilePath, JSON.stringify({ version: "1.0.0", invalidData: true }));
+
+    const configInstance = await setupConfigFile<any>(context.getProjectsPath(), {
+      context,
+      migrator: customMigrator,
+    });
+    const config = await configInstance.getConfig();
+
+    expect(config.version).toBe("2.0.0");
+
+    // Verify a timestamped corrupted backup file exists
+    const files = await fs.readdir(configDir);
+    const corruptedFile = files.find(
+      (f) => f.startsWith("projects.corrupted.") && f.endsWith(".json"),
+    );
+    expect(corruptedFile).toBeDefined();
+    const corruptedContent = JSON.parse(
+      await fs.readFile(path.join(configDir, corruptedFile!), "utf8"),
+    );
+    expect(corruptedContent.invalidData).toBe(true);
+  });
+
+  test("should save config to disk via setConfig", async () => {
+    const configInstance = await setupProjectsConfigFile(context);
+    const initialConfig = await configInstance.getConfig();
+
+    const newConfig: FileRepo = {
+      ...initialConfig,
+      pipelines: [
+        {
+          id: "pipeline-new",
+          project: "main",
+          type: "internal",
+          configName: "pipeline-new",
+          lastModified: "2026-06-05",
+        },
+      ],
+    };
+
+    const success = await configInstance.setConfig(newConfig);
+    expect(success).toBe(true);
+
+    const savedContent = JSON.parse(
+      await fs.readFile(context.getProjectsPath(), "utf8"),
+    );
+    expect(savedContent.pipelines).toHaveLength(1);
+    expect(savedContent.pipelines[0].id).toBe("pipeline-new");
+  });
+});
+
+describe("deletePipelineConfigFile", () => {
+  let tempDir: string;
+  let context: PipelabContext;
+
+  beforeEach(() => {
+    vol.reset();
+    tempDir = "/tmp/pipelab-test-config";
+    context = new PipelabContext({ userDataPath: tempDir });
+  });
+
+  test("should delete pipeline config file using relative name", async () => {
+    const configDir = context.getConfigPath();
+    await fs.mkdir(configDir, { recursive: true });
+
+    const pipelineFilePath = path.join(configDir, "my-pipeline.json");
+    await fs.writeFile(pipelineFilePath, JSON.stringify({ name: "My Pipeline" }));
+    expect(existsSync(pipelineFilePath)).toBe(true);
+
+    await deletePipelineConfigFileByName("my-pipeline", context);
+    expect(existsSync(pipelineFilePath)).toBe(false);
+  });
+
+  test("should delete pipeline config file using absolute path", async () => {
+    const configDir = context.getConfigPath();
+    await fs.mkdir(configDir, { recursive: true });
+
+    const pipelineFilePath = path.join(configDir, "another-pipeline.json");
+    await fs.writeFile(pipelineFilePath, JSON.stringify({ name: "Another Pipeline" }));
+    expect(existsSync(pipelineFilePath)).toBe(true);
+
+    await deletePipelineConfigFileByPath(pipelineFilePath, context);
+    expect(existsSync(pipelineFilePath)).toBe(false);
+  });
+
+  test("should not throw error when attempting to delete non-existent pipeline file", async () => {
+    const nonExistentPath = context.getConfigPath("does-not-exist.json");
+    expect(existsSync(nonExistentPath)).toBe(false);
+
+    await expect(deletePipelineConfigFileByName("does-not-exist", context)).resolves.not.toThrow();
+  });
+});
+

package/src/config.ts

@@ -2,36 +2,26 @@ import { PipelabContext } from "./context";
 import path from "node:path";
 import { ensure } from "./utils/fs-extras";
 import fs from "node:fs/promises";
-import { useLogger } from "@pipelab/shared";
-import { configRegistry, Migrator } from "@pipelab/shared";
-
-export const getMigrator = <T>(name: string) => {
-  if (configRegistry[name]) {
-    return configRegistry[name] as Migrator<T>;
-  }
-
-  if (name.startsWith("pipeline-")) {
-    return configRegistry["pipeline"] as Migrator<T>;
-  }
-
-  return undefined;
-};
+import {
+  useLogger,
+  Migrator,
+  AppConfig,
+  ConnectionsConfig,
+  FileRepo,
+  SavedFile,
+  appSettingsMigrator,
+  connectionsMigrator,
+  fileRepoMigrations,
+  savedFileMigrator,
+} from "@pipelab/shared";
 
 export const setupConfigFile = async <T>(
-  name: string,
-  options: { context: PipelabContext; migrator?: Migrator<T> },
+  filesPath: string,
+  options: { context: PipelabContext; migrator: Migrator<T> },
 ) => {
   const ctx = options.context;
-  const migrator = options.migrator || getMigrator<T>(name);
-
-  if (!migrator) {
-    throw new Error(
-      `No migrator found for configuration: ${name}. All managed files must have a migration schema.`,
-    );
-  }
-
-  const isAbsolutePath = path.isAbsolute(name);
-  const filesPath = isAbsolutePath ? name : ctx.getConfigPath(`${name}.json`);
+  const parsedPath = path.parse(filesPath);
+  const migrator = options.migrator;
 
   await ensure(filesPath, JSON.stringify(migrator.defaultValue));
 
@@ -42,7 +32,7 @@ export const setupConfigFile = async <T>(
         await fs.writeFile(filesPath, JSON.stringify(config));
         return true;
       } catch (e) {
-        logger().error(`Error saving config ${name}:`, e);
+        logger().error(`Error saving config ${parsedPath.name}:`, e);
         return false;
       }
     },
@@ -50,6 +40,7 @@ export const setupConfigFile = async <T>(
       const { logger } = useLogger();
       let content = undefined;
       let originalJson: any = undefined;
+      let parseFailed = false;
 
       try {
         content = await fs.readFile(filesPath, "utf8");
@@ -57,45 +48,66 @@ export const setupConfigFile = async <T>(
           originalJson = JSON.parse(content);
         }
       } catch (e) {
-        logger().error(`Error reading or parsing config ${name}:`, e);
+        logger().error(`Error reading or parsing config ${parsedPath.name}:`, e);
+        parseFailed = true;
       }
 
-      // console.log("content", content);
-
       let json: any = undefined;
+      let migrationFailed = false;
       try {
-        json = await migrator.migrate(originalJson, {
-          debug: false,
-        });
+        if (!parseFailed) {
+          json = await migrator.migrate(originalJson, {
+            debug: false,
+            onStep: async (state: any, version: string) => {
+              const versionedPath = path.join(parsedPath.dir, `${parsedPath.name}.v${version}.json`);
+              try {
+                await fs.writeFile(versionedPath, JSON.stringify(state));
+                logger().info(`Intermediate backup created for ${parsedPath.name} at ${versionedPath}`);
+              } catch (e) {
+                logger().error(
+                  `Failed to create intermediate backup for ${parsedPath.name} at v${version}:`,
+                  e,
+                );
+              }
+            },
+          });
+        } else {
+          json = migrator.defaultValue;
+        }
       } catch (e) {
-        logger().error(`Error migrating config ${name}:`, e);
+        logger().error(`Error migrating config ${parsedPath.name}:`, e);
+        migrationFailed = true;
         json = migrator.defaultValue;
       }
 
       const originalVersion = originalJson?.version;
       const newVersion = json?.version;
 
-      // Check if migration actually changed the version
-      if (originalVersion !== newVersion && content !== undefined) {
-        // Backup previous file before overwriting
-        const parsedPath = path.parse(filesPath);
-        const versionSuffix = originalVersion || "unknown";
-        const backupPath = path.join(parsedPath.dir, `${parsedPath.name}.${versionSuffix}.bak`);
-
-        try {
-          await fs.copyFile(filesPath, backupPath);
-          logger().info(`Backup created for ${name} at ${backupPath} (version ${versionSuffix})`);
-        } catch (e) {
-          logger().error(`Failed to create backup for ${name}:`, e);
+      const shouldSaveBack =
+        originalVersion !== newVersion || content === undefined || parseFailed || migrationFailed;
+
+      if (shouldSaveBack) {
+        if (parseFailed || migrationFailed) {
+          try {
+            const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+            const corruptedPath = path.join(
+              parsedPath.dir,
+              `${parsedPath.name}.corrupted.${timestamp}.json`,
+            );
+            const backupContent = parseFailed
+              ? content || ""
+              : JSON.stringify(originalJson, null, 2);
+            await fs.writeFile(corruptedPath, backupContent);
+            logger().info(`Corrupted config file preserved at ${corruptedPath}`);
+          } catch (e) {
+            logger().error(`Failed to backup corrupted config ${parsedPath.name}:`, e);
+          }
         }
-      }
 
-      // Save back migrated config if changed or if it's a new file
-      if (originalVersion !== newVersion || content === undefined) {
         try {
           await fs.writeFile(filesPath, JSON.stringify(json));
         } catch (e) {
-          logger().error(`Error saving migrated config ${name}:`, e);
+          logger().error(`Error saving migrated config ${parsedPath.name}:`, e);
         }
       }
 
@@ -103,3 +115,52 @@ export const setupConfigFile = async <T>(
     },
   };
 };
+
+export const setupSettingsConfigFile = (context: PipelabContext) => {
+  return setupConfigFile<AppConfig>(context.getSettingsPath(), {
+    context,
+    migrator: appSettingsMigrator,
+  });
+};
+
+export const setupConnectionsConfigFile = (context: PipelabContext) => {
+  return setupConfigFile<ConnectionsConfig>(context.getConnectionsPath(), {
+    context,
+    migrator: connectionsMigrator,
+  });
+};
+
+export const setupProjectsConfigFile = (context: PipelabContext) => {
+  return setupConfigFile<FileRepo>(context.getProjectsPath(), {
+    context,
+    migrator: fileRepoMigrations,
+  });
+};
+
+export const setupPipelineConfigFileByName = (name: string, context: PipelabContext) => {
+  const filesPath = context.getConfigPath(`${name}.json`);
+  return setupConfigFile<SavedFile>(filesPath, {
+    context,
+    migrator: savedFileMigrator,
+  });
+};
+
+export const setupPipelineConfigFileByPath = (absolutePath: string, context: PipelabContext) => {
+  return setupConfigFile<SavedFile>(absolutePath, {
+    context,
+    migrator: savedFileMigrator,
+  });
+};
+
+const deleteConfigFile = async (filesPath: string) => {
+  await fs.rm(filesPath, { force: true });
+};
+
+export const deletePipelineConfigFileByName = async (name: string, context: PipelabContext) => {
+  const filesPath = context.getConfigPath(`${name}.json`);
+  await deleteConfigFile(filesPath);
+};
+
+export const deletePipelineConfigFileByPath = async (absolutePath: string, context: PipelabContext) => {
+  await deleteConfigFile(absolutePath);
+};