@valbuild/server 0.12.0 → 0.13.3

package/src/ValFSHost.ts

@@ -0,0 +1,66 @@
+import path from "path";
+import ts from "typescript";
+import { ValFS } from "./ValFS";
+
+/**
+ * An implementation of methods in the various ts.*Host interfaces
+ * that uses ValFS to resolve modules and read/write files.
+ */
+export interface IValFSHost
+  extends ts.ParseConfigHost,
+    ts.ModuleResolutionHost {
+  useCaseSensitiveFileNames: boolean;
+
+  writeFile(fileName: string, text: string, encoding: "binary" | "utf8"): void;
+}
+
+export class ValFSHost implements IValFSHost {
+  constructor(
+    protected readonly valFS: ValFS,
+    protected readonly currentDirectory: string
+  ) {}
+
+  useCaseSensitiveFileNames = true;
+  readDirectory(
+    rootDir: string,
+    extensions: readonly string[],
+    excludes: readonly string[] | undefined,
+    includes: readonly string[],
+    depth?: number | undefined
+  ): readonly string[] {
+    return this.valFS.readDirectory(
+      rootDir,
+      extensions,
+      excludes,
+      includes,
+      depth
+    );
+  }
+
+  writeFile(fileName: string, text: string, encoding: "binary" | "utf8"): void {
+    this.valFS.writeFile(fileName, text, encoding);
+  }
+
+  getCurrentDirectory(): string {
+    return this.currentDirectory;
+  }
+
+  getCanonicalFileName(fileName: string): string {
+    if (path.isAbsolute(fileName)) {
+      return path.normalize(fileName);
+    }
+    return path.resolve(this.getCurrentDirectory(), fileName);
+  }
+
+  fileExists(fileName: string): boolean {
+    return this.valFS.fileExists(fileName);
+  }
+
+  readFile(fileName: string): string | undefined {
+    return this.valFS.readFile(fileName);
+  }
+
+  realpath(path: string): string {
+    return this.valFS.realpath(path);
+  }
+}

package/src/ValModuleLoader.test.ts

@@ -0,0 +1,75 @@
+import { createModuleLoader } from "./ValModuleLoader";
+import path from "path";
+
+const TestCaseDir = "../test/example-projects";
+const TestCases = [
+  { name: "basic-next-typescript", valConfigDir: ".", ext: "ts" },
+  {
+    name: "basic-next-src-typescript",
+    valConfigDir: "./src",
+    ext: "ts",
+  },
+  { name: "basic-next-javascript", valConfigDir: ".", ext: "js" },
+  { name: "typescript-description-files", valConfigDir: ".", ext: "js" },
+];
+
+describe("val module loader", () => {
+  test.each(TestCases)(
+    "resolution and smoke test transpilation for: $name",
+    async (testCase) => {
+      const rootDir = path.resolve(__dirname, TestCaseDir, testCase.name);
+      const loader = createModuleLoader(rootDir);
+      expect(
+        await loader.getModule(
+          loader.resolveModulePath(
+            `${testCase.valConfigDir}/val-system.${testCase.ext}`,
+            "./pages/blogs.val"
+          )
+        )
+      ).toContain("/pages/blogs");
+      expect(
+        await loader.getModule(
+          loader.resolveModulePath(
+            `${testCase.valConfigDir}/pages/blogs.val.${testCase.ext}`,
+            "../val.config"
+          )
+        )
+      ).toContain("@valbuild/core");
+    }
+  );
+
+  test("resolution based on baseDir / paths in tsconfig", () => {
+    const rootDir = path.resolve(
+      __dirname,
+      TestCaseDir,
+      "basic-next-src-typescript"
+    );
+    const moduleLoader = createModuleLoader(rootDir);
+
+    const containingFile = "./src/pages/blogs.val.ts";
+    const baseCase = moduleLoader.resolveModulePath(
+      containingFile,
+      "../val.config"
+    ); // tsconfig maps @ to src
+    const pathsMapping = moduleLoader.resolveModulePath(
+      containingFile,
+      "@/val.config"
+    ); // tsconfig maps @ to src
+    expect(baseCase).toBeDefined();
+    expect(baseCase).toEqual(pathsMapping);
+  });
+
+  test("resolution on .d.ts files", () => {
+    const rootDir = path.resolve(
+      __dirname,
+      TestCaseDir,
+      "typescript-description-files"
+    );
+    const moduleLoader = createModuleLoader(rootDir);
+
+    const containingFile = "./pages/blogs.val.js";
+    expect(
+      moduleLoader.resolveModulePath(containingFile, "../val.config")
+    ).toMatch(/val\.config\.js$/);
+  });
+});

package/src/ValModuleLoader.ts

@@ -0,0 +1,128 @@
+import ts from "typescript";
+import { getCompilerOptions } from "./getCompilerOptions";
+import type { IValFSHost } from "./ValFSHost";
+import { ValSourceFileHandler } from "./ValSourceFileHandler";
+import fs from "fs";
+
+const JsFileLookupMapping: [resolvedFileExt: string, replacements: string[]][] =
+  [
+    // NOTE: first one matching will be used
+    [".cjs.d.ts", [".esm.js", ".mjs.js"]],
+    [".cjs.js", [".esm.js", ".mjs.js"]],
+    [".cjs", [".mjs"]],
+    [".d.ts", [".js"]],
+  ];
+
+export const createModuleLoader = (
+  rootDir: string,
+  host: IValFSHost = {
+    ...ts.sys,
+    writeFile: fs.writeFileSync,
+  }
+): ValModuleLoader => {
+  const compilerOptions = getCompilerOptions(rootDir, host);
+  const sourceFileHandler = new ValSourceFileHandler(
+    rootDir,
+    compilerOptions,
+    host
+  );
+  const loader = new ValModuleLoader(
+    rootDir,
+    compilerOptions,
+    sourceFileHandler,
+    host
+  );
+  return loader;
+};
+
+export class ValModuleLoader {
+  constructor(
+    public readonly projectRoot: string,
+    private readonly compilerOptions: ts.CompilerOptions,
+    private readonly sourceFileHandler: ValSourceFileHandler,
+    private readonly host: IValFSHost = {
+      ...ts.sys,
+      writeFile: fs.writeFileSync,
+    }
+  ) {}
+
+  getModule(modulePath: string): string {
+    if (!modulePath) {
+      throw Error(`Illegal module path: "${modulePath}"`);
+    }
+    const code = this.host.readFile(modulePath);
+    if (!code) {
+      throw Error(`Could not read file "${modulePath}"`);
+    }
+    return ts.transpile(code, {
+      ...this.compilerOptions,
+      // allowJs: true,
+      // rootDir: this.compilerOptions.rootDir,
+      module: ts.ModuleKind.ESNext,
+      target: ts.ScriptTarget.ES2020, // QuickJS supports a lot of ES2020: https://test262.report/, however not all cases are in that report (e.g. export const {} = {})
+      // moduleResolution: ts.ModuleResolutionKind.NodeNext,
+      // target: ts.ScriptTarget.ES2020, // QuickJs runs in ES2020 so we must use that
+    });
+  }
+
+  resolveModulePath(
+    containingFilePath: string,
+    requestedModuleName: string
+  ): string {
+    const sourceFileName = this.sourceFileHandler.resolveSourceModulePath(
+      containingFilePath,
+      requestedModuleName
+    );
+    const matches = this.findMatchingJsFile(sourceFileName);
+    if (matches.match === false) {
+      throw Error(
+        `Could not find matching js file for module "${requestedModuleName}". Tried:\n${matches.tried.join(
+          "\n"
+        )}`
+      );
+    }
+    const filePath = matches.match;
+    // resolve all symlinks (preconstruct for example symlinks the dist folder)
+    const followedPath = this.host.realpath?.(filePath) ?? filePath;
+    if (!followedPath) {
+      throw Error(
+        `File path was empty: "${filePath}", containing file: "${containingFilePath}", requested module: "${requestedModuleName}"`
+      );
+    }
+    return followedPath;
+  }
+
+  private findMatchingJsFile(
+    filePath: string
+  ): { match: string } | { match: false; tried: string[] } {
+    let requiresReplacements = false;
+    for (const [currentEnding] of JsFileLookupMapping) {
+      if (filePath.endsWith(currentEnding)) {
+        requiresReplacements = true;
+        break;
+      }
+    }
+    // avoid unnecessary calls to fileExists if we don't need to replace anything
+    if (!requiresReplacements) {
+      if (this.host.fileExists(filePath)) {
+        return { match: filePath };
+      }
+    }
+    const tried = [];
+    for (const [currentEnding, replacements] of JsFileLookupMapping) {
+      if (filePath.endsWith(currentEnding)) {
+        for (const replacement of replacements) {
+          const newFilePath =
+            filePath.slice(0, -currentEnding.length) + replacement;
+          if (this.host.fileExists(newFilePath)) {
+            return { match: newFilePath };
+          } else {
+            tried.push(newFilePath);
+          }
+        }
+      }
+    }
+
+    return { match: false, tried: tried.concat(filePath) };
+  }
+}

package/src/ValQuickJSRuntime.ts

@@ -0,0 +1,47 @@
+import { JSModuleNormalizeResult, QuickJSWASMModule } from "quickjs-emscripten";
+import { ValModuleLoader } from "./ValModuleLoader";
+
+export async function newValQuickJSRuntime(
+  quickJSModule: Pick<QuickJSWASMModule, "newRuntime">,
+  moduleLoader: ValModuleLoader,
+  {
+    maxStackSize = 1024 * 640, // TODO: these were randomly chosen, we should figure out what the right values are:
+    memoryLimit = 1024 * 640,
+  }: {
+    maxStackSize?: number;
+    memoryLimit?: number;
+  } = {}
+) {
+  const runtime = quickJSModule.newRuntime();
+
+  runtime.setMaxStackSize(maxStackSize);
+  runtime.setMemoryLimit(memoryLimit);
+
+  runtime.setModuleLoader(
+    (modulePath) => {
+      try {
+        return { value: moduleLoader.getModule(modulePath) };
+      } catch (e) {
+        return {
+          error: Error(`Could not resolve module: ${modulePath}'`),
+        };
+      }
+    },
+    (baseModuleName, requestedName): JSModuleNormalizeResult => {
+      try {
+        const modulePath = moduleLoader.resolveModulePath(
+          baseModuleName,
+          requestedName
+        );
+        return { value: modulePath };
+      } catch (e) {
+        console.debug(
+          `Could not resolve ${requestedName} in ${baseModuleName}`,
+          e
+        );
+        return { value: requestedName };
+      }
+    }
+  );
+  return runtime;
+}

package/src/ValServer.ts

@@ -0,0 +1,23 @@
+import express from "express";
+
+export interface ValServer {
+  authorize(req: express.Request, res: express.Response): Promise<void>;
+
+  callback(req: express.Request, res: express.Response): Promise<void>;
+
+  logout(req: express.Request, res: express.Response): Promise<void>;
+
+  session(req: express.Request, res: express.Response): Promise<void>;
+
+  getIds(
+    req: express.Request<{ 0: string }>,
+    res: express.Response
+  ): Promise<void>;
+
+  patchIds(
+    req: express.Request<{ 0: string }>,
+    res: express.Response
+  ): Promise<void>;
+
+  commit(req: express.Request, res: express.Response): Promise<void>;
+}

package/src/ValSourceFileHandler.ts

@@ -0,0 +1,57 @@
+import ts from "typescript";
+import path from "path";
+import { IValFSHost } from "./ValFSHost";
+import fs from "fs";
+
+export class ValSourceFileHandler {
+  constructor(
+    private readonly projectRoot: string,
+    private readonly compilerOptions: ts.CompilerOptions,
+    private readonly host: IValFSHost = {
+      ...ts.sys,
+      writeFile: fs.writeFileSync,
+    }
+  ) {}
+
+  getSourceFile(filePath: string): ts.SourceFile | undefined {
+    const fileContent = this.host.readFile(filePath);
+    const scriptTarget = this.compilerOptions.target ?? ts.ScriptTarget.ES2020;
+    if (fileContent) {
+      return ts.createSourceFile(filePath, fileContent, scriptTarget);
+    }
+  }
+
+  writeSourceFile(sourceFile: ts.SourceFile) {
+    return this.writeFile(sourceFile.fileName, sourceFile.text, "utf8");
+  }
+
+  writeFile(filePath: string, content: string, encoding: "binary" | "utf8") {
+    this.host.writeFile(filePath, content, encoding);
+  }
+
+  resolveSourceModulePath(
+    containingFilePath: string,
+    requestedModuleName: string
+  ): string {
+    const resolutionRes = ts.resolveModuleName(
+      requestedModuleName,
+      path.isAbsolute(containingFilePath)
+        ? containingFilePath
+        : path.resolve(this.projectRoot, containingFilePath),
+      this.compilerOptions,
+      this.host,
+      undefined,
+      undefined,
+      ts.ModuleKind.ESNext
+    );
+    const resolvedModule = resolutionRes.resolvedModule;
+    if (!resolvedModule) {
+      throw Error(
+        `Could not resolve module "${requestedModuleName}", base: "${containingFilePath}": No resolved modules returned: ${JSON.stringify(
+          resolutionRes
+        )}`
+      );
+    }
+    return resolvedModule.resolvedFileName;
+  }
+}

package/src/createRequestHandler.ts

@@ -0,0 +1,24 @@
+import express, { RequestHandler, Router } from "express";
+import { ValServer } from "./ValServer";
+import { createRequestHandler as createUIRequestHandler } from "@valbuild/ui/server";
+
+export function createRequestHandler(valServer: ValServer): RequestHandler {
+  const router = Router();
+
+  router.use("/static", createUIRequestHandler());
+  router.get("/session", valServer.session.bind(valServer));
+  router.get("/authorize", valServer.authorize.bind(valServer));
+  router.get("/callback", valServer.callback.bind(valServer));
+  router.get("/logout", valServer.logout.bind(valServer));
+  router.get<{ 0: string }>("/ids/*", valServer.getIds.bind(valServer));
+  router.patch<{ 0: string }>(
+    "/ids/*",
+    express.json({
+      type: "application/json-patch+json",
+      limit: "10mb",
+    }),
+    valServer.patchIds.bind(valServer)
+  );
+  router.post("/commit", valServer.commit.bind(valServer));
+  return router;
+}

package/src/expressHelpers.ts

@@ -0,0 +1,5 @@
+import { SourcePath } from "@valbuild/core/src/val";
+
+export function getPathFromParams(params: { 0: string }): SourcePath {
+  return `/${params[0]}` as SourcePath;
+}

package/src/getCompilerOptions.ts

@@ -0,0 +1,50 @@
+import path from "path";
+import ts from "typescript";
+
+export const getCompilerOptions = (
+  rootDir: string,
+  parseConfigHost: ts.ParseConfigHost
+): ts.CompilerOptions => {
+  const tsConfigPath = path.resolve(rootDir, "tsconfig.json");
+  const jsConfigPath = path.resolve(rootDir, "jsconfig.json");
+  let configFilePath: string;
+  if (parseConfigHost.fileExists(jsConfigPath)) {
+    configFilePath = jsConfigPath;
+  } else if (parseConfigHost.fileExists(tsConfigPath)) {
+    configFilePath = tsConfigPath;
+  } else {
+    throw Error(
+      `Could not read config from: "${tsConfigPath}" nor "${jsConfigPath}". Root dir: "${rootDir}"`
+    );
+  }
+  const { config, error } = ts.readConfigFile(
+    configFilePath,
+    parseConfigHost.readFile.bind(parseConfigHost)
+  );
+  if (error) {
+    if (typeof error.messageText === "string") {
+      throw Error(
+        `Could not parse config file: ${configFilePath}. Error: ${error.messageText}`
+      );
+    }
+    throw Error(
+      `Could not parse config file: ${configFilePath}. Error: ${error.messageText.messageText}`
+    );
+  }
+  const optionsOverrides = undefined;
+  const parsedConfigFile = ts.parseJsonConfigFileContent(
+    config,
+    parseConfigHost,
+    rootDir,
+    optionsOverrides,
+    configFilePath
+  );
+  if (parsedConfigFile.errors.length > 0) {
+    throw Error(
+      `Could not parse config file: ${configFilePath}. Errors: ${parsedConfigFile.errors
+        .map((e) => e.messageText)
+        .join("\n")}`
+    );
+  }
+  return parsedConfigFile.options;
+};

package/src/hosting.ts

@@ -0,0 +1,156 @@
+import type { RequestListener } from "node:http";
+import express from "express";
+import { createService, ServiceOptions } from "./Service";
+import { createRequestHandler } from "./createRequestHandler";
+import { ValServer } from "./ValServer";
+import { LocalValServer, LocalValServerOptions } from "./LocalValServer";
+import { ProxyValServer, ProxyValServerOptions } from "./ProxyValServer";
+
+type Opts = ValServerOverrides & ServiceOptions;
+
+type ValServerOverrides = Partial<{
+  /**
+   * Override the Val API key.
+   *
+   * Typically this is set using VAL_API_KEY env var.
+   *
+   * NOTE: if this is set you must also set valSecret or VAL_SECRET env var.
+   */
+  apiKey: string;
+  /**
+   * Override the Val session key.
+   *
+   * This can be any randomly generated string.
+   * It will be used for authentication between the frontend and val api
+   * endpoints in this app.
+   *
+   * Typically this is set using VAL_SECRET env var.
+   *
+   * NOTE: if this is set you must also set apiKey or VAL_API_KEY env var.
+   */
+  valSecret: string;
+  /**
+   * Override the default the mode of operation.
+   *
+   * Typically this should not be set.
+   *
+   * "local" means that changes will be written to the local filesystem,
+   * which is what you want when developing locally.
+   *
+   * "proxy" means that changes will proxied to https://app.val.build
+   * and eventually be committed in the Git repository.
+   *
+   * It will automatically be "proxy" if both VAL_API_KEY env var (or the apiKey property) and VAL_SECRET env var (or the valSecret property)
+   * is set.
+   *
+   * If both is missing, it will default to "local".
+   */
+  mode: "proxy" | "local";
+  /**
+   * Current git commit.
+   *
+   * Required if mode is "proxy".
+   *
+   * @example "e83c5163316f89bfbde7d9ab23ca2e25604af290"
+   */
+  gitCommit: string;
+
+  /**
+   * Current git branch.
+   *
+   * Required if mode is "proxy".
+   *
+   * @example "main"
+   */
+  gitBranch: string;
+  /**
+   * The base url of Val.
+   *
+   * Typically this should not be set.
+   *
+   * Can also be overridden using the VAL_BUILD_URL env var.
+   *
+   * @example "https://app.val.build"
+   */
+  valBuildUrl: string;
+}>;
+
+async function _createRequestListener(
+  route: string,
+  opts: Opts
+): Promise<RequestListener> {
+  const serverOpts = await initHandlerOptions(route, opts);
+  let valServer: ValServer;
+  if (serverOpts.mode === "proxy") {
+    valServer = new ProxyValServer(serverOpts);
+  } else {
+    valServer = new LocalValServer(serverOpts);
+  }
+  const reqHandler = createRequestHandler(valServer);
+  return express().use(route, reqHandler);
+}
+
+type ValServerOptions =
+  | ({ mode: "proxy" } & ProxyValServerOptions)
+  | ({ mode: "local" } & LocalValServerOptions);
+async function initHandlerOptions(
+  route: string,
+  opts: ValServerOverrides & ServiceOptions
+): Promise<ValServerOptions> {
+  const maybeApiKey = opts.apiKey || process.env.VAL_API_KEY;
+  const maybeValSecret = opts.valSecret || process.env.VAL_SECRET;
+  const isProxyMode =
+    opts.mode === "proxy" ||
+    (opts.mode === undefined && (maybeApiKey || maybeValSecret));
+
+  if (isProxyMode) {
+    const valBuildUrl =
+      opts.valBuildUrl || process.env.VAL_BUILD_URL || "https://app.val.build";
+    if (!maybeApiKey || !maybeValSecret) {
+      throw new Error(
+        "VAL_API_KEY and VAL_SECRET env vars must both be set in proxy mode"
+      );
+    }
+    const maybeGitCommit = opts.gitCommit || process.env.VAL_GIT_COMMIT;
+    if (!maybeGitCommit) {
+      throw new Error("VAL_GIT_COMMIT env var must be set in proxy mode");
+    }
+    const maybeGitBranch = opts.gitBranch || process.env.VAL_GIT_BRANCH;
+    if (!maybeGitBranch) {
+      throw new Error("VAL_GIT_BRANCH env var must be set in proxy mode");
+    }
+    return {
+      mode: "proxy",
+      route,
+      apiKey: maybeApiKey,
+      valSecret: maybeValSecret,
+      valBuildUrl,
+      gitCommit: maybeGitCommit,
+      gitBranch: maybeGitBranch,
+    };
+  } else {
+    const service = await createService(process.cwd(), opts);
+    return {
+      mode: "local",
+      service,
+    };
+  }
+}
+
+// TODO: rename to createValApiHandlers?
+export function createRequestListener(
+  route: string,
+  opts: Opts
+): RequestListener {
+  const handler = _createRequestListener(route, opts);
+  return async (req, res) => {
+    try {
+      return (await handler)(req, res);
+    } catch (e) {
+      res.statusCode = 500;
+      res.write(e instanceof Error ? e.message : "Unknown error");
+      res.end();
+      return;
+    }
+  };
+}

package/src/index.ts

@@ -0,0 +1,12 @@
+export type { ServiceOptions } from "./Service";
+export { createService, Service } from "./Service";
+export { createRequestHandler } from "./createRequestHandler";
+export { createRequestListener } from "./hosting";
+export { ValModuleLoader } from "./ValModuleLoader";
+export { getCompilerOptions } from "./getCompilerOptions";
+export { ValSourceFileHandler } from "./ValSourceFileHandler";
+export { ValFSHost } from "./ValFSHost";
+export type { IValFSHost } from "./ValFSHost";
+export type { ValFS } from "./ValFS";
+export { patchSourceFile } from "./patchValFile";
+export { formatSyntaxErrorTree } from "./patch/ts/syntax";