@valbuild/cli 0.72.4 → 0.73.1

package/src/login.ts

@@ -0,0 +1,112 @@
+import pc from "picocolors";
+import fs from "fs";
+import path from "path";
+import { getPersonalAccessTokenPath } from "@valbuild/server";
+
+const host = process.env.VAL_BUILD_URL || "https://app.val.build";
+
+export async function login(options: { root?: string }) {
+  try {
+    console.log(pc.cyan("\nStarting login process...\n"));
+
+    // Step 1: Initiate login and get token and URL
+    const response = await fetch(`${host}/api/login`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+    });
+    let token;
+    let url;
+    if (!response.headers.get("content-type")?.includes("application/json")) {
+      const text = await response.text();
+      console.error(
+        pc.red(
+          "Unexpected failure while trying to login (content type was not JSON). Server response:",
+        ),
+        text || "<empty>",
+      );
+      process.exit(1);
+    }
+    const json = await response.json();
+    if (json) {
+      token = json.nonce;
+      url = json.url;
+    }
+    if (!token || !url) {
+      console.error(pc.red("Unexpected response from the server."), json);
+      process.exit(1);
+    }
+
+    console.log(pc.green("Open the following URL in your browser to log in:"));
+    console.log(pc.underline(pc.blue(url)));
+    console.log(pc.dim("\nWaiting for login confirmation...\n"));
+
+    // Step 2: Poll for login confirmation
+    const result = await pollForConfirmation(token);
+
+    // Step 3: Save the token
+    const filePath = getPersonalAccessTokenPath(options.root || process.cwd());
+    saveToken(result, filePath);
+  } catch (error) {
+    console.error(
+      pc.red(
+        "An error occurred during the login process. Check your internet connection. Details:",
+      ),
+      error instanceof Error ? error.message : JSON.stringify(error, null, 2),
+    );
+    process.exit(1);
+  }
+}
+
+const MAX_DURATION = 5 * 60 * 1000; // 5 minutes
+async function pollForConfirmation(token: string): Promise<{
+  profile: { username: string };
+  pat: string;
+}> {
+  const start = Date.now();
+  while (Date.now() - start < MAX_DURATION) {
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    const response = await fetch(
+      `${host}/api/login?token=${token}&consume=true`,
+    );
+    if (response.status === 500) {
+      console.error(pc.red("An error occurred on the server."));
+      process.exit(1);
+    }
+    if (response.status === 200) {
+      const json = await response.json();
+      if (json) {
+        if (
+          typeof json.profile.username === "string" &&
+          typeof json.pat === "string"
+        ) {
+          return json;
+        } else {
+          console.error(pc.red("Unexpected response from the server."));
+          process.exit(1);
+        }
+      }
+    }
+  }
+  console.error(pc.red("Login confirmation timed out."));
+  process.exit(1);
+}
+
+function saveToken(
+  result: {
+    profile: { username: string };
+    pat: string;
+  },
+  filePath: string,
+) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, JSON.stringify(result, null, 2));
+  console.log(
+    pc.green(
+      `Token for ${pc.cyan(
+        result.profile.username,
+      )} saved to ${pc.cyan(filePath)}`,
+    ),
+  );
+}

package/src/utils/evalValConfigFile.ts

@@ -0,0 +1,90 @@
+import path from "path";
+import fs from "fs/promises";
+import vm from "node:vm";
+import ts from "typescript"; // TODO: make this dependency optional (only required if the file is val.config.ts not val.config.js)
+import z from "zod";
+import { ValConfig } from "@valbuild/core";
+
+const ValConfigSchema = z.object({
+  project: z.string().optional(),
+  root: z.string().optional(),
+  files: z
+    .object({
+      directory: z
+        .string()
+        .refine((val): val is `/public/val` => val.startsWith("/public/val"), {
+          message: "files.directory must start with '/public/val'",
+        }),
+    })
+    .optional(),
+  gitCommit: z.string().optional(),
+  gitBranch: z.string().optional(),
+  defaultTheme: z.union([z.literal("light"), z.literal("dark")]).optional(),
+  ai: z
+    .object({
+      commitMessages: z
+        .object({
+          disabled: z.boolean().optional(),
+        })
+        .optional(),
+    })
+    .optional(),
+});
+
+export async function evalValConfigFile(
+  projectRoot: string,
+  configFileName: string,
+): Promise<ValConfig | null> {
+  const valConfigPath = path.join(projectRoot, configFileName);
+
+  let code: string | null = null;
+  try {
+    code = await fs.readFile(valConfigPath, "utf-8");
+  } catch (err) {
+    //
+  }
+  if (!code) {
+    return null;
+  }
+
+  const transpiled = ts.transpileModule(code, {
+    compilerOptions: {
+      target: ts.ScriptTarget.ES2020,
+      module: ts.ModuleKind.CommonJS,
+      esModuleInterop: true,
+    },
+    fileName: valConfigPath,
+  });
+
+  const exportsObj = {};
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const sandbox: Record<string, any> = {
+    exports: exportsObj,
+    module: { exports: exportsObj },
+    require, // NOTE: this is a security risk, but this code is running in the users own environment at the CLI level
+    __filename: valConfigPath,
+    __dirname: path.dirname(valConfigPath),
+    console,
+    process,
+  };
+  sandbox.global = sandbox;
+
+  const context = vm.createContext(sandbox);
+  const script = new vm.Script(transpiled.outputText, {
+    filename: valConfigPath,
+  });
+  script.runInContext(context);
+  const valConfig = sandbox.module.exports.config;
+  if (!valConfig) {
+    throw Error(
+      `Val config file at path: '${valConfigPath}' must export a config object. Got: ${valConfig}`,
+    );
+  }
+  const result = ValConfigSchema.safeParse(valConfig);
+  if (!result.success) {
+    throw Error(
+      `Val config file at path: '${valConfigPath}' has invalid schema: ${result.error.message}`,
+    );
+  }
+  return result.data;
+}

package/src/utils/getFileExt.ts

@@ -0,0 +1,4 @@
+export function getFileExt(filePath: string) {
+  // NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
+  return filePath.split(".").pop() || "";
+}

package/src/utils/getValCoreVersion.ts

@@ -0,0 +1,5 @@
+import { getVersions } from "../getVersions";
+
+export function getValCoreVersion() {
+  return getVersions().coreVersion || "unknown";
+}

package/src/validate.ts

@@ -1,10 +1,20 @@
 import path from "path";
-import { createFixPatch, createService } from "@valbuild/server";
 import {
+  createFixPatch,
+  createService,
+  getSettings,
+  getPersonalAccessTokenPath,
+  parsePersonalAccessTokenFile,
+  uploadRemoteFile,
+} from "@valbuild/server";
+import {
+  DEFAULT_VAL_REMOTE_HOST,
   FILE_REF_PROP,
   Internal,
   ModuleFilePath,
   ModulePath,
+  SerializedFileSchema,
+  SerializedImageSchema,
   SourcePath,
   ValidationFix,
 } from "@valbuild/core";
@@ -12,6 +22,7 @@ import { glob } from "fast-glob";
 import picocolors from "picocolors";
 import { ESLint } from "eslint";
 import fs from "fs/promises";
+import { evalValConfigFile } from "./utils/evalValConfigFile";
 
 export async function validate({
   root,
@@ -22,11 +33,20 @@ export async function validate({
   fix?: boolean;
   noEslint?: boolean;
 }) {
+  const valRemoteHost = process.env.VAL_REMOTE_HOST || DEFAULT_VAL_REMOTE_HOST;
   const projectRoot = root ? path.resolve(root) : process.cwd();
   const eslint = new ESLint({
     cwd: projectRoot,
     ignore: false,
   });
+  const valConfigFile =
+    (await evalValConfigFile(projectRoot, "val.config.ts")) ||
+    (await evalValConfigFile(projectRoot, "val.config.js"));
+  console.log(
+    picocolors.greenBright(
+      `Validating project${valConfigFile?.project ? ` '${picocolors.inverse(valConfigFile?.project)}'` : ""}...`,
+    ),
+  );
   const service = await createService(projectRoot, {});
   const checkKeyIsValid = async (
     key: string,
@@ -119,8 +139,8 @@ export async function validate({
       "files",
     );
   }
-  console.log("Validating...", valFiles.length, "files");
-
+  console.log(picocolors.greenBright(`Found ${valFiles.length} files...`));
+  let publicProjectId: string | undefined;
   let didFix = false; // TODO: ugly
   async function validateFile(file: string): Promise<number> {
     const moduleFilePath = `/${file}` as ModuleFilePath; // TODO: check if this always works? (Windows?)
@@ -135,6 +155,12 @@ export async function validate({
       "utf-8",
     );
     const eslintResult = eslintResultsByFile?.[file];
+    const remoteFiles: Record<
+      SourcePath,
+      { ref: string; metadata?: Record<string, unknown> }
+    > = {};
+    let remoteFileBuckets: string[] | null = null;
+    let remoteFilesCounter = 0;
     eslintResult?.messages.forEach((m) => {
       // display surrounding code
       logEslintMessage(fileContent, moduleFilePath, m);
@@ -152,6 +178,7 @@ export async function validate({
           (prev, m) => (m.severity >= 2 ? prev + 1 : prev),
           0,
         ) || 0;
+      let fixedErrors = 0;
       if (valModule.errors) {
         if (valModule.errors.validation) {
           for (const [sourcePath, validationErrors] of Object.entries(
@@ -178,24 +205,31 @@ export async function validate({
                       valModule.source,
                       valModule.schema,
                     );
-                    const filePath = path.join(
-                      projectRoot,
-                      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                      (fileSource.source as any)?.[FILE_REF_PROP],
-                    );
+                    let filePath: string | null = null;
                     try {
+                      filePath = path.join(
+                        projectRoot,
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        (fileSource.source as any)?.[FILE_REF_PROP],
+                      );
                       await fs.access(filePath);
                     } catch {
-                      console.log(
-                        picocolors.red("✘"),
-                        `File ${filePath} does not exist`,
-                      );
+                      if (filePath) {
+                        console.log(
+                          picocolors.red("✘"),
+                          `File ${filePath} does not exist`,
+                        );
+                      } else {
+                        console.log(
+                          picocolors.red("✘"),
+                          `Expected file to be defined at: ${sourcePath} but no file was found`,
+                        );
+                      }
                       errors += 1;
                       continue;
                     }
                   }
                 } else if (v.fixes.includes("keyof:check-keys")) {
-                  const prevErrors = errors;
                   if (
                     v.value &&
                     typeof v.value === "object" &&
@@ -238,31 +272,274 @@ export async function validate({
                     );
                     errors += 1;
                   }
-                  if (prevErrors < errors) {
+                } else if (
+                  v.fixes.includes("image:upload-remote") ||
+                  v.fixes.includes("file:upload-remote")
+                ) {
+                  if (!fix) {
+                    console.log(
+                      picocolors.red("✘"),
+                      `Remote file ${sourcePath} needs to be uploaded (use --fix to upload)`,
+                    );
+                    errors += 1;
+                    continue;
+                  }
+                  const [, modulePath] =
+                    Internal.splitModuleFilePathAndModulePath(
+                      sourcePath as SourcePath,
+                    );
+                  if (valModule.source && valModule.schema) {
+                    const resolvedRemoteFileAtSourcePath = Internal.resolvePath(
+                      modulePath,
+                      valModule.source,
+                      valModule.schema,
+                    );
+                    let filePath: string | null = null;
+                    try {
+                      filePath = path.join(
+                        projectRoot,
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        (resolvedRemoteFileAtSourcePath.source as any)?.[
+                          FILE_REF_PROP
+                        ],
+                      );
+                      await fs.access(filePath);
+                    } catch {
+                      if (filePath) {
+                        console.log(
+                          picocolors.red("✘"),
+                          `File ${filePath} does not exist`,
+                        );
+                      } else {
+                        console.log(
+                          picocolors.red("✘"),
+                          `Expected file to be defined at: ${sourcePath} but no file was found`,
+                        );
+                      }
+                      errors += 1;
+                      continue;
+                    }
+                    const patFile = getPersonalAccessTokenPath(projectRoot);
+                    try {
+                      await fs.access(patFile);
+                    } catch {
+                      // TODO: display this error only once:
+                      console.log(
+                        picocolors.red("✘"),
+                        `File: ${path.join(projectRoot, file)} has remote images that are not uploaded and you are not logged in.\n\nFix this error by logging in:\n\t"npx val login"\n`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+
+                    const parsedPatFile = parsePersonalAccessTokenFile(
+                      await fs.readFile(patFile, "utf-8"),
+                    );
+                    if (!parsedPatFile.success) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Error parsing personal access token file: ${parsedPatFile.error}. You need to login again.`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    const { pat } = parsedPatFile.data;
+
+                    if (remoteFiles[sourcePath as SourcePath]) {
+                      console.log(
+                        picocolors.yellow("⚠"),
+                        `Remote file ${filePath} already uploaded`,
+                      );
+                      continue;
+                    }
+                    // TODO: parallelize this:
+                    console.log(
+                      picocolors.yellow("⚠"),
+                      `Uploading remote file ${filePath}...`,
+                    );
+
+                    if (!resolvedRemoteFileAtSourcePath.schema) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Cannot upload remote file: schema not found for ${sourcePath}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+
+                    const actualRemoteFileSource =
+                      resolvedRemoteFileAtSourcePath.source;
+                    const fileSourceMetadata = Internal.isFile(
+                      actualRemoteFileSource,
+                    )
+                      ? actualRemoteFileSource.metadata
+                      : undefined;
+                    const resolveRemoteFileSchema =
+                      resolvedRemoteFileAtSourcePath.schema;
+                    if (!resolveRemoteFileSchema) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Could not resolve schema for remote file: ${sourcePath}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    if (!publicProjectId || !remoteFileBuckets) {
+                      let projectName = process.env.VAL_PROJECT;
+                      if (!projectName) {
+                        projectName = valConfigFile?.project;
+                      }
+                      if (!projectName) {
+                        console.log(
+                          picocolors.red("✘"),
+                          "Project name not found. Set VAL_PROJECT environment variable or add project name to val.config",
+                        );
+                        errors += 1;
+                        continue;
+                      }
+                      const settingsRes = await getSettings(projectName, {
+                        pat,
+                      });
+                      if (!settingsRes.success) {
+                        console.log(
+                          picocolors.red("✘"),
+                          `Could not get public project id: ${settingsRes.message}.`,
+                        );
+                        errors += 1;
+                        continue;
+                      }
+                      publicProjectId = settingsRes.data.publicProjectId;
+                      remoteFileBuckets =
+                        settingsRes.data.remoteFileBuckets.map((b) => b.bucket);
+                    }
+                    if (!publicProjectId) {
+                      console.log(
+                        picocolors.red("✘"),
+                        "Could not get public project id",
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    if (
+                      resolveRemoteFileSchema.type !== "image" &&
+                      resolveRemoteFileSchema.type !== "file"
+                    ) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `The schema is the remote is neither image nor file: ${sourcePath}`,
+                      );
+                    }
+                    remoteFilesCounter += 1;
+                    const bucket =
+                      remoteFileBuckets[
+                        remoteFilesCounter % remoteFileBuckets.length
+                      ];
+                    if (!bucket) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Internal error: could not allocate a bucket for the remote file located at ${sourcePath}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    let fileBuffer: Buffer;
+                    try {
+                      fileBuffer = await fs.readFile(filePath);
+                    } catch (e) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Error reading file: ${e}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    const relativeFilePath = path
+                      .relative(projectRoot, filePath)
+                      .split(path.sep)
+                      .join("/") as `public/val/${string}`;
+                    if (!relativeFilePath.startsWith("public/val/")) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `File path must be within the public/val/ directory (e.g. public/val/path/to/file.txt). Got: ${relativeFilePath}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    const remoteFileUpload = await uploadRemoteFile(
+                      valRemoteHost,
+                      fileBuffer,
+                      publicProjectId,
+                      bucket,
+                      relativeFilePath,
+                      resolveRemoteFileSchema as
+                        | SerializedFileSchema
+                        | SerializedImageSchema,
+                      fileSourceMetadata,
+                      { pat },
+                    );
+                    if (!remoteFileUpload.success) {
+                      console.log(
+                        picocolors.red("✘"),
+                        `Error uploading remote file: ${remoteFileUpload.error}`,
+                      );
+                      errors += 1;
+                      continue;
+                    }
+                    console.log(
+                      picocolors.yellow("⚠"),
+                      `Uploaded remote file ${filePath}`,
+                    );
+                    remoteFiles[sourcePath as SourcePath] = {
+                      ref: remoteFileUpload.ref,
+                      metadata: fileSourceMetadata,
+                    };
+                  }
+                } else if (
+                  v.fixes.includes("image:download-remote") ||
+                  v.fixes.includes("file:download-remote")
+                ) {
+                  if (fix) {
+                    console.log(
+                      picocolors.yellow("⚠"),
+                      `Downloading remote file in ${sourcePath}...`,
+                    );
+                  } else {
                     console.log(
                       picocolors.red("✘"),
-                      "Found error in",
-                      `${sourcePath}`,
+                      `Remote file ${sourcePath} needs to be downloaded (use --fix to download)`,
                     );
+                    errors += 1;
+                    continue;
                   }
+                } else if (
+                  v.fixes.includes("image:check-remote") ||
+                  v.fixes.includes("file:check-remote")
+                ) {
+                  // skip
                 } else {
                   console.log(
                     picocolors.red("✘"),
-                    "Found error in",
-                    `${sourcePath}:`,
-                    v.message,
+                    "Unknown fix",
+                    v.fixes,
+                    "for",
+                    sourcePath,
                   );
                   errors += 1;
+                  continue;
                 }
                 const fixPatch = await createFixPatch(
-                  { projectRoot },
+                  { projectRoot, remoteHost: valRemoteHost },
                   !!fix,
                   sourcePath as SourcePath,
                   v,
+                  remoteFiles,
+                  valModule.source,
+                  valModule.schema,
                 );
                 if (fix && fixPatch?.patch && fixPatch?.patch.length > 0) {
                   await service.patch(moduleFilePath, fixPatch.patch);
                   didFix = true;
+                  fixedErrors += 1;
                   console.log(
                     picocolors.yellow("⚠"),
                     "Applied fix for",
@@ -272,8 +549,10 @@ export async function validate({
                 fixPatch?.remainingErrors?.forEach((e) => {
                   errors += 1;
                   console.log(
-                    v.fixes ? picocolors.yellow("⚠") : picocolors.red("✘"),
-                    `Found ${v.fixes ? "fixable " : ""}error in`,
+                    e.fixes && e.fixes.length
+                      ? picocolors.yellow("⚠")
+                      : picocolors.red("✘"),
+                    `Got ${e.fixes && e.fixes.length ? "fixable " : ""}error in`,
                     `${sourcePath}:`,
                     e.message,
                   );
@@ -282,7 +561,7 @@ export async function validate({
                 errors += 1;
                 console.log(
                   picocolors.red("✘"),
-                  "Found error in",
+                  "Got error in",
                   `${sourcePath}:`,
                   v.message,
                 );
@@ -290,6 +569,16 @@ export async function validate({
             }
           }
         }
+        if (
+          fixedErrors === errors &&
+          (!valModule.errors.fatal || valModule.errors.fatal.length == 0)
+        ) {
+          console.log(
+            picocolors.green("✔"),
+            moduleFilePath,
+            "is valid (" + (Date.now() - start) + "ms)",
+          );
+        }
         for (const fatalError of valModule.errors.fatal || []) {
           errors += 1;
           console.log(
@@ -306,6 +595,13 @@ export async function validate({
           "is valid (" + (Date.now() - start) + "ms)",
         );
       }
+      if (errors > 0) {
+        console.log(
+          picocolors.red("✘"),
+          `${`/${file}`} contains ${errors} error${errors > 1 ? "s" : ""}`,
+          " (" + (Date.now() - start) + "ms)",
+        );
+      }
       return errors;
     }
   }
@@ -325,9 +621,9 @@ export async function validate({
   if (errors > 0) {
     console.log(
       picocolors.red("✘"),
-      "Found",
+      "Got",
       errors,
-      "validation error" + (errors > 1 ? "s" : ""),
+      "error" + (errors > 1 ? "s" : ""),
     );
     process.exit(1);
   } else {