@pipelab/core-node 1.0.0-beta.0

package/src/utils/remote.ts

@@ -0,0 +1,497 @@
+import { dirname, delimiter, join } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdir, readdir, readFile, writeFile, access, chmod, rm, cp } from "node:fs/promises";
+import { existsSync, constants } from "node:fs";
+import pacote from "pacote";
+import semver from "semver";
+import { isDev, projectRoot, PipelabContext } from "../context";
+import { execa } from "execa";
+import { sendStartupProgress } from "../server";
+import { downloadFile, extractZip, extractTarGz, generateTempFolder } from "./fs-extras";
+
+export const DEFAULT_NODE_VERSION = "24.14.1";
+export const DEFAULT_PNPM_VERSION = "10.12.0";
+
+/**
+ * In-memory lock to prevent concurrent operations on the same resource (e.g., downloading Node.js).
+ */
+const activeOperations = new Map<string, Promise<any>>();
+const packumentRequests = new Map<string, Promise<any>>();
+
+async function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
+  const existing = activeOperations.get(key);
+  if (existing) {
+    console.log(`[Lock] Waiting for concurrent operation on: ${key}`);
+    return existing;
+  }
+
+  const promise = (async () => {
+    try {
+      return await fn();
+    } finally {
+      activeOperations.delete(key);
+    }
+  })();
+
+  activeOperations.set(key, promise);
+  return promise;
+}
+
+export type FetchOptions = {
+  installDeps?: boolean;
+  signal?: AbortSignal;
+  context: PipelabContext;
+};
+
+/**
+ * Robust utility to fetch, cache, and resolve an NPM package.
+ * Centralized in core-node to avoid circular dependencies.
+ */
+export async function fetchPackage(
+  packageName: string,
+  versionOrRange: string,
+  options: FetchOptions,
+): Promise<{
+  packageDir: string;
+  resolvedVersion: string;
+  isLocal?: boolean;
+  entryPoint?: string;
+}> {
+  // 0. Check for local monorepo package in development
+  if (isDev && projectRoot && process.env.PIPELAB_FORCE_NPM !== "true") {
+    if (packageName.startsWith("@pipelab/")) {
+      const local = await tryResolveMonorepoPackage(packageName);
+      if (local) {
+        console.log(`[Fetcher] ${packageName}: Resolved to local source at ${local.packageDir}`);
+        return {
+          ...local,
+          resolvedVersion: "workspace",
+        };
+      }
+    }
+  }
+
+  const ctx = options.context;
+  const baseDir = ctx.getPackagesPath(packageName);
+  let resolvedVersion: string;
+
+  console.log(`[Fetcher] Resolving ${packageName}@${versionOrRange || "latest"}...`);
+
+  try {
+    // 1. Resolve version/range using npm with session-wide memoization and disk cache
+    const cachePath = join(ctx.userDataPath, "cache", "pacote");
+    let packumentPromise = packumentRequests.get(packageName);
+    if (!packumentPromise) {
+      packumentPromise = pacote.packument(packageName, { cache: cachePath });
+      packumentRequests.set(packageName, packumentPromise);
+    }
+
+    const packument = await packumentPromise;
+    const versions = Object.keys(packument.versions);
+    const range = versionOrRange || "latest";
+
+    // Prioritize tags (like 'latest', 'beta', etc.) over semver ranges
+    const foundVersion = packument["dist-tags"]?.[range] || semver.maxSatisfying(versions, range);
+
+    if (!foundVersion) {
+      throw new Error(`Package ${packageName}@${range} not found on npm (available tags: ${Object.keys(packument["dist-tags"] || {}).join(", ")})`);
+    }
+    resolvedVersion = foundVersion;
+    console.log(`[Fetcher] ${packageName}: Resolved to v${resolvedVersion} via npm`);
+  } catch (error) {
+    console.warn(`[Fetcher] ${packageName}: remote resolution failed, trying local fallback...`);
+    const fallbackVersion = await tryLocalFallback(versionOrRange, error, baseDir, packageName);
+    if (fallbackVersion) {
+      resolvedVersion = fallbackVersion;
+    } else {
+      throw error;
+    }
+  }
+
+  const cachePath = join(ctx.userDataPath, "cache", "pacote");
+  const packageDir = join(baseDir, resolvedVersion);
+
+  // If the package already exists and we don't need to install dependencies, return immediately
+  if (existsSync(packageDir) && !options?.installDeps) {
+    return { packageDir, resolvedVersion };
+  }
+
+  const lockKey = `package:${packageName}:${resolvedVersion}`;
+  return withLock(lockKey, async () => {
+    if (!existsSync(packageDir)) {
+      console.log(`[Fetcher] ${packageName}@${resolvedVersion}: Downloading to ${packageDir}...`);
+      await mkdir(packageDir, { recursive: true });
+      await pacote.extract(`${packageName}@${resolvedVersion}`, packageDir, {
+        cache: cachePath,
+      });
+    }
+
+    // 2. Resolve entry point from package.json for downloaded package
+    const entryPoint = await resolveEntryPoint(packageDir, packageName);
+
+    if (options?.installDeps) {
+      await installDependencies(packageDir, packageName, options);
+    }
+
+    return { packageDir, resolvedVersion, entryPoint };
+  });
+}
+
+/**
+ * Executes a pnpm install in a specific directory with a portable environment.
+ */
+export async function runPnpm(
+  cwd: string,
+  options: {
+    args?: string[];
+    extraEnv?: Record<string, string>;
+    signal?: AbortSignal;
+    context: PipelabContext;
+  },
+) {
+  const {
+    args = ["install", "--prod", "--no-lockfile", "--prefer-offline", "--no-verify-store-integrity"],
+    extraEnv = {},
+    signal,
+    context: ctx,
+  } = options;
+
+  const [nodePath, pnpmPath] = await Promise.all([
+    ensureNodeJS(ctx).catch(() => process.execPath),
+    ensurePNPM(ctx).catch(() => "pnpm"),
+  ]);
+
+  const isScript = pnpmPath.endsWith(".cjs") || pnpmPath.endsWith(".js");
+  const command = isScript ? nodePath : pnpmPath;
+  const finalArgs = isScript ? [pnpmPath, ...args] : args;
+
+  return execa(command, finalArgs, {
+    cwd,
+    all: true,
+    cancelSignal: signal,
+    env: {
+      ...process.env,
+      NODE_ENV: "production",
+      PATH: nodePath ? `${dirname(nodePath)}${delimiter}${process.env.PATH}` : process.env.PATH,
+      PNPM_HOME: join(ctx.userDataPath, "pnpm"),
+      PNPM_ONLY_ALLOW_TRUSTED_DEPENDENCIES: "false",
+      ...extraEnv,
+    },
+  });
+}
+
+/**
+ * Installs a specific version of Node.js if not already present.
+ */
+export async function ensureNodeJS(
+  context: PipelabContext,
+  version = DEFAULT_NODE_VERSION,
+) {
+  const isWindows = process.platform === "win32";
+  const nodeDir = context.getThirdPartyPath("node", version);
+  const finalNodePath = join(nodeDir, isWindows ? "node.exe" : "bin/node");
+
+  if (existsSync(finalNodePath)) {
+    return finalNodePath;
+  }
+
+  const lockKey = `node:${version}`;
+  return withLock(lockKey, async () => {
+    if (existsSync(finalNodePath)) return finalNodePath;
+
+    const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : "x86";
+    const platform = isWindows ? "win" : process.platform === "darwin" ? "osx" : "linux";
+    const extension = isWindows ? "zip" : "tar.gz";
+    const downloadPlatform = platform === "osx" ? "darwin" : platform;
+
+    const fileName = `node-v${version}-${downloadPlatform}-${arch}.${extension}`;
+    const downloadUrl = `https://nodejs.org/dist/v${version}/${fileName}`;
+    const tempDir = await generateTempFolder(tmpdir());
+    const archivePath = join(tempDir, fileName);
+
+    sendStartupProgress(`Downloading Node.js v${version}...`);
+    console.log(`Downloading Node.js from ${downloadUrl}...`);
+    await downloadFile(downloadUrl, archivePath);
+
+    sendStartupProgress(`Extracting Node.js v${version}...`);
+    console.log(`Extracting Node.js to ${tempDir}...`);
+    const extractTempDir = join(tempDir, "extracted");
+    await mkdir(extractTempDir, { recursive: true });
+
+    if (extension === "zip") {
+      await extractZip(archivePath, extractTempDir);
+    } else {
+      await extractTarGz(archivePath, extractTempDir);
+    }
+
+    const extractedEntries = await readdir(extractTempDir);
+    const nodeSubDir = extractedEntries.find((entry) => entry.startsWith(`node-v${version}`));
+    if (!nodeSubDir) throw new Error(`Could not find extracted Node.js directory`);
+
+    const sourceDir = join(extractTempDir, nodeSubDir);
+    await mkdir(dirname(nodeDir), { recursive: true });
+    await rm(nodeDir, { recursive: true, force: true });
+    await cp(sourceDir, nodeDir, { recursive: true });
+    await rm(tempDir, { recursive: true, force: true });
+
+    if (!isWindows) await chmod(finalNodePath, 0o755).catch(() => { });
+    return finalNodePath;
+  });
+}
+
+/**
+ * Installs the PNPM package from npm if not already present.
+ */
+export async function ensurePNPM(
+  context: PipelabContext,
+  version = DEFAULT_PNPM_VERSION,
+) {
+  const pnpmDir = context.getPackagesPath("pnpm", version);
+  const pnpmPath = join(pnpmDir, "bin", "pnpm.cjs");
+
+  if (existsSync(pnpmPath)) {
+    return pnpmPath;
+  }
+
+  const lockKey = `pnpm:${version}`;
+  return withLock(lockKey, async () => {
+    if (existsSync(pnpmPath)) return pnpmPath;
+    sendStartupProgress(`Checking PNPM v${version}...`);
+    const { packageDir } = await fetchPackage("pnpm", version, {
+      context,
+    });
+    return join(packageDir, "bin", "pnpm.cjs");
+  });
+}
+
+async function installDependencies(packageDir: string, packageName: string, options: FetchOptions) {
+  const nodeModulesPath = join(packageDir, "node_modules");
+
+  if (existsSync(nodeModulesPath)) {
+    try {
+      const files = await readdir(nodeModulesPath);
+      if (files.length === 0) {
+        console.warn(`[Fetcher] ${packageName}: node_modules exists but is empty. Re-installing...`);
+      }
+    } catch (e) {
+      // Continue to install if readdir fails
+    }
+  }
+
+  console.log(`[Fetcher] ${packageName}: Ensuring dependencies are installed...`);
+  try {
+    const { all } = await runPnpm(packageDir, {
+      signal: options.signal,
+      context: options.context,
+    });
+
+    if (all) console.log(`[Fetcher] ${packageName}: Installation trace:\n${all}`);
+    console.log(`[Fetcher] ${packageName}: Dependencies installed successfully.`);
+  } catch (err: any) {
+    console.error(
+      `[Fetcher] ${packageName}: CRITICAL ERROR during dependency installation: ${err.message}`,
+    );
+    if (err.all) console.error(`[Fetcher] ${packageName}: Error details:\n${err.all}`);
+    throw new Error(`Failed to install dependencies for ${packageName}. See logs for details.`);
+  }
+}
+
+export async function fetchPipelabAsset(
+  packageName: string,
+  versionOrRange: string,
+  options: FetchOptions,
+): Promise<string> {
+  if (isDev && projectRoot) {
+    const assetId = packageName.replace("@pipelab/asset-", "");
+    const localPath = join(projectRoot, "assets", `asset-${assetId}`);
+    if (existsSync(localPath)) return localPath;
+  }
+  const { packageDir } = await fetchPackage(packageName, versionOrRange, options);
+  return packageDir;
+}
+
+export async function fetchPipelabPlugin(
+  pluginName: string,
+  versionOrRange: string,
+  options: FetchOptions,
+): Promise<{ packageDir: string; entryPoint: string; isLocal: boolean }> {
+  const { packageDir, isLocal, entryPoint } = await fetchPackage(pluginName, versionOrRange, {
+    installDeps: true,
+    ...options,
+  });
+
+  // Default entry point if not provided by fetchPackage
+  let finalEntryPoint = entryPoint;
+  if (!finalEntryPoint) {
+    const patterns = [
+      join(packageDir, "dist", "index.mjs"),
+      join(packageDir, "index.mjs"),
+    ];
+    finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
+  }
+
+  return { packageDir, entryPoint: finalEntryPoint, isLocal: !!isLocal };
+}
+
+export async function fetchPipelabCli(
+  versionOrRange: string,
+  options: FetchOptions,
+): Promise<{ packageDir: string; entryPoint: string; isLocal: boolean }> {
+  const { packageDir, isLocal, entryPoint } = await fetchPackage(
+    "@pipelab/cli",
+    versionOrRange,
+    options,
+  );
+
+  // Default entry point for CLI if not provided
+  let finalEntryPoint = entryPoint;
+  if (!finalEntryPoint) {
+    const patterns = [
+      join(packageDir, "dist", "index.mjs"),
+      join(packageDir, "index.mjs"),
+    ];
+    finalEntryPoint = patterns.find((p) => existsSync(p)) || patterns[0];
+  }
+
+  return { packageDir, entryPoint: finalEntryPoint, isLocal: !!isLocal };
+}
+
+/**
+ * Cache for monorepo package locations to avoid repeated disk crawling.
+ */
+let monorepoCache: Record<string, string> | null = null;
+
+async function tryResolveMonorepoPackage(
+  packageName: string,
+): Promise<{ packageDir: string; isLocal: boolean; entryPoint: string } | null> {
+  if (!monorepoCache) {
+    monorepoCache = await crawlMonorepoPackages();
+  }
+
+  const packageDir = monorepoCache[packageName];
+  if (!packageDir) return null;
+
+  // Find best entry point from package.json
+  let entryPoint: string | undefined;
+  try {
+    const pkgPath = join(packageDir, "package.json");
+    const pkg = JSON.parse(await readFile(pkgPath, "utf-8"));
+
+    // 1. Try to find the entry point from package.json
+    // User tip: main is usually source in dev, publishConfig.main is compiled for prod
+    const publishMain = pkg.publishConfig?.module || pkg.publishConfig?.main;
+    const devMain = pkg.module || pkg.main;
+
+    // Check bin field if it's a CLI
+    const binField =
+      typeof pkg.bin === "string"
+        ? pkg.bin
+        : pkg.bin?.[packageName.replace("@pipelab/", "")] ||
+        (pkg.bin ? pkg.bin[Object.keys(pkg.bin)[0]] : undefined);
+
+    // In dev, we prefer the "main" field if it points to TS, or a hardcoded src/index.ts
+    const tsSource = join(packageDir, "src", "index.ts");
+
+    if (devMain && devMain.endsWith(".ts")) {
+      entryPoint = join(packageDir, devMain);
+    } else if (existsSync(tsSource)) {
+      entryPoint = tsSource;
+    } else if (binField) {
+      entryPoint = join(packageDir, binField);
+    } else if (publishMain) {
+      entryPoint = join(packageDir, publishMain);
+    } else if (devMain) {
+      entryPoint = join(packageDir, devMain);
+    }
+  } catch (e) {
+    console.warn(`[Fetcher] Failed to parse package.json for ${packageName}:`, e);
+  }
+
+  return {
+    packageDir,
+    isLocal: true,
+    entryPoint: entryPoint || join(packageDir, "dist", "index.mjs"),
+  };
+}
+
+async function crawlMonorepoPackages(): Promise<Record<string, string>> {
+  const cache: Record<string, string> = {};
+  if (!projectRoot) return cache;
+
+  const searchDirs = ["plugins", "packages", "apps"];
+  for (const dir of searchDirs) {
+    const fullDir = join(projectRoot, dir);
+    if (!existsSync(fullDir)) continue;
+
+    try {
+      const entries = await readdir(fullDir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.isDirectory()) {
+          const pkgPath = join(fullDir, entry.name, "package.json");
+          if (existsSync(pkgPath)) {
+            try {
+              const pkg = JSON.parse(await readFile(pkgPath, "utf-8"));
+              if (pkg.name) {
+                cache[pkg.name] = join(fullDir, entry.name);
+              }
+            } catch (e) { }
+          }
+        }
+      }
+    } catch (e) { }
+  }
+  return cache;
+}
+
+async function findLatestLocalVersion(baseDir: string): Promise<string | null> {
+  if (!existsSync(baseDir)) return null;
+  try {
+    const entries = await readdir(baseDir, { withFileTypes: true });
+    const versions = entries
+      .filter((e) => e.isDirectory() || e.isSymbolicLink())
+      .map((e) => e.name)
+      .sort((a, b) => b.localeCompare(a, undefined, { numeric: true, sensitivity: "base" }));
+    return versions[0] || null;
+  } catch {
+    return null;
+  }
+}
+
+async function tryLocalFallback(
+  _version: string | undefined,
+  _error: unknown,
+  baseDir: string,
+  logPrefix: string,
+): Promise<string | null> {
+  const latestLocal = await findLatestLocalVersion(baseDir);
+  if (latestLocal) {
+    console.info(`[Fetcher] ${logPrefix}: Using locally cached version: ${latestLocal}`);
+    return latestLocal;
+  }
+  return null;
+}
+
+/**
+ * Resolves the entry point of a package by looking at its package.json.
+ */
+async function resolveEntryPoint(packageDir: string, packageName: string) {
+  try {
+    const pkgPath = join(packageDir, "package.json");
+    if (!existsSync(pkgPath)) return undefined;
+
+    const pkg = JSON.parse(await readFile(pkgPath, "utf-8"));
+    const main = pkg.module || pkg.main || pkg.publishConfig?.module || pkg.publishConfig?.main;
+
+    if (main) {
+      return join(packageDir, main);
+    }
+
+    if (pkg.bin) {
+      const binFile = typeof pkg.bin === "string" ? pkg.bin : Object.values(pkg.bin)[0];
+      if (binFile) return join(packageDir, binFile as string);
+    }
+  } catch (e) {
+    console.warn(`[Fetcher] ${packageName}: Failed to resolve entry point:`, e);
+  }
+  return undefined;
+}

package/src/utils/storage.ts

@@ -0,0 +1,99 @@
+import fs from "node:fs";
+import path from "node:path";
+import { useLogger } from "@pipelab/shared";
+import { PipelabContext } from "../context";
+
+/**
+ * A generic file-based storage implementation for persisting JSON data.
+ * Useful for Supabase Auth and other CLI-specific persistent data.
+ */
+export class JsonFileStorage {
+  private filePath: string;
+  private logger = useLogger().logger;
+
+  constructor(fileName: string, context: PipelabContext) {
+    this.filePath = path.join(context.userDataPath, fileName);
+    const dir = path.dirname(this.filePath);
+    if (!fs.existsSync(dir)) {
+      try {
+        fs.mkdirSync(dir, { recursive: true });
+      } catch (e) {
+        this.logger().error(`[Storage] Failed to create directory ${dir}:`, e);
+        throw e;
+      }
+    }
+  }
+
+  getItem(key: string): string | null {
+    try {
+      if (!fs.existsSync(this.filePath)) return null;
+      const content = fs.readFileSync(this.filePath, "utf8");
+      if (!content) return null;
+      const data = JSON.parse(content);
+      return data[key] || null;
+    } catch (e) {
+      this.logger().error(`[Storage] Failed to read or parse ${this.filePath}:`, e);
+      // We throw here because the user explicitly asked NOT to ignore errors
+      throw e;
+    }
+  }
+
+  setItem(key: string, value: string): void {
+    try {
+      let data: any = {};
+      if (fs.existsSync(this.filePath)) {
+        const content = fs.readFileSync(this.filePath, "utf8");
+        if (content) {
+          data = JSON.parse(content);
+        }
+      }
+      data[key] = value;
+      fs.writeFileSync(this.filePath, JSON.stringify(data, null, 2), "utf8");
+    } catch (e) {
+      this.logger().error(`[Storage] Failed to write to ${this.filePath}:`, e);
+      throw e;
+    }
+  }
+
+  removeItem(key: string): void {
+    try {
+      if (!fs.existsSync(this.filePath)) return;
+      const content = fs.readFileSync(this.filePath, "utf8");
+      if (!content) return;
+      const data = JSON.parse(content);
+      delete data[key];
+      fs.writeFileSync(this.filePath, JSON.stringify(data, null, 2), "utf8");
+    } catch (e) {
+      this.logger().error(`[Storage] Failed to remove key from ${this.filePath}:`, e);
+      throw e;
+    }
+  }
+
+  /**
+   * Helper to get the entire data object
+   */
+  getAll(): Record<string, any> {
+    try {
+      if (!fs.existsSync(this.filePath)) return {};
+      const content = fs.readFileSync(this.filePath, "utf8");
+      return content ? JSON.parse(content) : {};
+    } catch (e) {
+      this.logger().error(`[Storage] Failed to read all data from ${this.filePath}:`, e);
+      throw e;
+    }
+  }
+
+  /**
+   * Helper to clear the entire storage file
+   */
+  clear(): void {
+    try {
+      if (fs.existsSync(this.filePath)) {
+        fs.unlinkSync(this.filePath);
+      }
+    } catch (e) {
+      this.logger().error(`[Storage] Failed to clear ${this.filePath}:`, e);
+      throw e;
+    }
+  }
+}