@valbuild/server 0.21.1 → 0.22.0

package/dist/declarations/src/LocalValServer.d.ts

@@ -23,4 +23,5 @@ export declare class LocalValServer implements ValServer {
     authorize(req: express.Request, res: express.Response): Promise<void>;
     callback(req: express.Request, res: express.Response): Promise<void>;
     logout(req: express.Request, res: express.Response): Promise<void>;
+    getFiles(req: express.Request, res: express.Response): Promise<void>;
 }

package/dist/declarations/src/SerializedModuleContent.d.ts

@@ -1,6 +1,7 @@
 import { type Source, type SerializedSchema, ValidationErrors } from "@valbuild/core";
 import { ModuleId, type SourcePath } from "@valbuild/core/src/val";
 export declare const FATAL_ERROR_TYPES: readonly ["no-schema", "no-source", "invalid-id", "no-module"];
+export type FatalErrorType = (typeof FATAL_ERROR_TYPES)[number];
 export type SerializedModuleContent = {
     source: Source;
     schema: SerializedSchema;
@@ -9,14 +10,14 @@ export type SerializedModuleContent = {
 } | {
     source?: Source;
     schema?: SerializedSchema;
-    path?: SourcePath;
+    path: SourcePath;
     errors: {
         invalidModuleId?: ModuleId;
         validation?: ValidationErrors;
         fatal?: {
             message: string;
-            stack?: string[];
-            type?: (typeof FATAL_ERROR_TYPES)[number];
+            stack?: string;
+            type?: FatalErrorType;
         }[];
     };
 };

package/dist/declarations/src/Service.d.ts

@@ -9,9 +9,15 @@ export type ServiceOptions = {
     /**
      * Relative path to the val.config.js file from the root directory.
      *
-     * @example src/val.config
+     * @example "./val.config"
      */
     valConfigPath: string;
+    /**
+     * Disable cache for transpilation
+     *
+     * @default false
+     *    */
+    disableCache?: boolean;
 };
 export declare function createService(projectRoot: string, opts: ServiceOptions, host?: IValFSHost, loader?: ValModuleLoader): Promise<Service>;
 export declare class Service {

package/dist/declarations/src/ValModuleLoader.d.ts

@@ -7,7 +7,10 @@ export declare class ValModuleLoader {
     private readonly compilerOptions;
     private readonly sourceFileHandler;
     private readonly host;
-    constructor(projectRoot: string, compilerOptions: ts.CompilerOptions, sourceFileHandler: ValSourceFileHandler, host?: IValFSHost);
+    private readonly disableCache;
+    private cache;
+    private cacheSize;
+    constructor(projectRoot: string, compilerOptions: ts.CompilerOptions, sourceFileHandler: ValSourceFileHandler, host?: IValFSHost, disableCache?: boolean);
     getModule(modulePath: string): string;
     resolveModulePath(containingFilePath: string, requestedModuleName: string): string;
     private findMatchingJsFile;

package/dist/declarations/src/ValServer.d.ts

@@ -11,4 +11,5 @@ export interface ValServer {
     enable(req: express.Request, res: express.Response): Promise<void>;
     disable(req: express.Request, res: express.Response): Promise<void>;
     getTree(req: express.Request, res: express.Response): Promise<void>;
+    getFiles(req: express.Request, res: express.Response): Promise<void>;
 }

package/dist/valbuild-server.cjs.dev.js

@@ -9,10 +9,12 @@ var core = require('@valbuild/core');
 var patch = require('@valbuild/core/patch');
 var path = require('path');
 var fs = require('fs');
+var sucrase = require('sucrase');
 var express = require('express');
 var server = require('@valbuild/ui/server');
 var z = require('zod');
 var crypto = require('crypto');
+var stream = require('stream');
 var sizeOf = require('image-size');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
@@ -640,6 +642,7 @@ globalThis.valModule = {
       const error = result.error.consume(context.dump);
       console.error(`Fatal error reading val file: ${error.message}\n`, error.stack);
       return {
+        path: id,
         errors: {
           invalidModuleId: id,
           fatal: [{
@@ -655,12 +658,12 @@ globalThis.valModule = {
         fatalErrors.push(`Could not find any modules at: ${id}`);
       } else {
         if (valModule.id !== id) {
-          fatalErrors.push(`Expected val id: '${id}' but got: '${valModule.id}'`);
+          fatalErrors.push(`Wrong val.content id! In the file of with: '${id}', found: '${valModule.id}'`);
         }
         if (!(valModule !== null && valModule !== void 0 && valModule.schema)) {
           fatalErrors.push(`Expected val id: '${id}' to have a schema`);
         }
-        if (!(valModule !== null && valModule !== void 0 && valModule.source)) {
+        if ((valModule === null || valModule === void 0 ? void 0 : valModule.source) === undefined) {
           fatalErrors.push(`Expected val id: '${id}' to have a source`);
         }
       }
@@ -680,7 +683,7 @@ globalThis.valModule = {
         };
       }
       return {
-        path: valModule.id,
+        path: valModule.id || id,
         // NOTE: we use path here, since SerializedModuleContent (maybe bad name?) can be used for whole modules as well as subparts of modules
         source: valModule.source,
         schema: valModule.schema,
@@ -734,9 +737,6 @@ const patchValFile = async (id, valConfigPath, patch$1, sourceFileHandler, runti
       sourceFileHandler.writeFile("." + filePath, convertDataUrlToBase64(content).toString("binary"), "binary");
     }
   }
-  for (const [ref, patch] of Object.entries(derefRes.value.remotePatches)) {
-    throw Error(`Cannot update remote ${ref} with ${JSON.stringify(patch)}: not implemented`);
-  }
   sourceFileHandler.writeSourceFile(newSourceFile.value);
   return readValFile(id, valConfigPath, runtime);
 };
@@ -815,15 +815,23 @@ class ValSourceFileHandler {
 const JsFileLookupMapping = [
 // NOTE: first one matching will be used
 [".cjs.d.ts", [".esm.js", ".mjs.js"]], [".cjs.js", [".esm.js", ".mjs.js"]], [".cjs", [".mjs"]], [".d.ts", [".js", ".esm.js", ".mjs.js"]]];
+const MAX_CACHE_SIZE = 100 * 1024 * 1024; // 100 mb
+const MAX_OBJECT_KEY_SIZE = 2 ** 27; // https://stackoverflow.com/questions/13367391/is-there-a-limit-on-length-of-the-key-string-in-js-object
+
 class ValModuleLoader {
-  constructor(projectRoot, compilerOptions, sourceFileHandler, host = {
+  constructor(projectRoot,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  compilerOptions, sourceFileHandler, host = {
     ...ts__default["default"].sys,
     writeFile: fs__default["default"].writeFileSync
-  }) {
+  }, disableCache = false) {
     this.projectRoot = projectRoot;
     this.compilerOptions = compilerOptions;
     this.sourceFileHandler = sourceFileHandler;
     this.host = host;
+    this.disableCache = disableCache;
+    this.cache = {};
+    this.cacheSize = 0;
   }
   getModule(modulePath) {
     if (!modulePath) {
@@ -833,18 +841,31 @@ class ValModuleLoader {
     if (!code) {
       throw Error(`Could not read file "${modulePath}"`);
     }
-    return ts__default["default"].transpile(code, {
-      ...this.compilerOptions,
-      jsx: ts__default["default"].JsxEmit.React,
-      // allowJs: true,
-      // rootDir: this.compilerOptions.rootDir,
-      module: ts__default["default"].ModuleKind.ESNext,
-      target: ts__default["default"].ScriptTarget.ES2015 // QuickJS supports a lot of ES2020: https://test262.report/, however not all cases are in that report (e.g. export const {} = {})
-      // moduleResolution: ts.ModuleResolutionKind.NodeNext,
-      // target: ts.ScriptTarget.ES2020, // QuickJs runs in ES2020 so we must use that
-    });
-  }
+    let compiledCode;
+    if (this.cache[code] && !this.disableCache) {
+      // TODO: use hash instead of code as key
+      compiledCode = this.cache[code];
+    } else {
+      compiledCode = sucrase.transform(code, {
+        filePath: modulePath,
+        disableESTransforms: true,
+        transforms: ["typescript"]
+      }).code;
+      if (!this.disableCache) {
+        if (this.cacheSize > MAX_CACHE_SIZE) {
+          console.warn("Cache size exceeded, clearing cache");
+          this.cache = {};
+          this.cacheSize = 0;
+        }
+        if (code.length < MAX_OBJECT_KEY_SIZE) {
+          this.cache[code] = compiledCode;
+          this.cacheSize += code.length + compiledCode.length; // code is mostly ASCII so 1 byte per char
+        }
+      }
+    }
 
+    return compiledCode;
+  }
   resolveModulePath(containingFilePath, requestedModuleName) {
     var _this$host$realpath, _this$host;
     let sourceFileName = this.sourceFileHandler.resolveSourceModulePath(containingFilePath, requestedModuleName);
@@ -1004,11 +1025,9 @@ class Service {
         path: sourcePath,
         schema: resolved.schema instanceof core.Schema ? resolved.schema.serialize() : resolved.schema,
         source: resolved.source,
-        errors: valModule.errors && valModule.errors.validation && valModule.errors.validation[sourcePath] ? {
-          validation: valModule.errors.validation[sourcePath] ? {
-            [sourcePath]: valModule.errors.validation[sourcePath]
-          } : undefined,
-          fatal: valModule.errors && valModule.errors.fatal ? valModule.errors.fatal : undefined
+        errors: valModule.errors && valModule.errors.validation ? {
+          validation: valModule.errors.validation || undefined,
+          fatal: valModule.errors.fatal || undefined
         } : false
       };
     } else {
@@ -1038,13 +1057,10 @@ function createRequestHandler(valServer) {
   router.get("/enable", valServer.enable.bind(valServer));
   router.get("/disable", valServer.disable.bind(valServer));
   router.get("/tree/*", valServer.getTree.bind(valServer));
+  router.get("/files/*", valServer.getFiles.bind(valServer));
   return router;
 }
 
-function getPathFromParams(params) {
-  return `/${params[0]}`;
-}
-
 const JSONValueT = z__default["default"].lazy(() => z__default["default"].union([z__default["default"].string(), z__default["default"].number(), z__default["default"].boolean(), z__default["default"].null(), z__default["default"].array(JSONValueT), z__default["default"].record(JSONValueT)]));
 
 /**
@@ -1082,6 +1098,7 @@ const OperationJSONT = z__default["default"].discriminatedUnion("op", [z__defaul
 }).strict(), z__default["default"].object({
   op: z__default["default"].literal("file"),
   path: z__default["default"].string(),
+  filePath: z__default["default"].string(),
   value: z__default["default"].string()
 }).strict()]);
 const PatchJSON = z__default["default"].array(OperationJSONT);
@@ -1148,10 +1165,61 @@ function encodeJwt(payload, sessionKey) {
 const VAL_SESSION_COOKIE = core.Internal.VAL_SESSION_COOKIE;
 const VAL_STATE_COOKIE = core.Internal.VAL_STATE_COOKIE;
 const VAL_ENABLED_COOKIE = core.Internal.VAL_ENABLE_COOKIE_NAME;
+class BrowserReadableStreamWrapper extends stream.Readable {
+  constructor(readableStream) {
+    super();
+    this.reader = readableStream.getReader();
+  }
+  _read() {
+    this.reader.read().then(({
+      done,
+      value
+    }) => {
+      if (done) {
+        this.push(null); // No more data to read
+      } else {
+        this.push(Buffer.from(value));
+      }
+    }).catch(error => {
+      this.emit("error", error);
+    });
+  }
+}
 class ProxyValServer {
   constructor(options) {
     this.options = options;
   }
+  async getFiles(req, res) {
+    return this.withAuth(req, res, async data => {
+      const url = new URL(`/v1/files/${this.options.valName}/${req.params["0"]}`, this.options.valContentUrl);
+      if (typeof req.query.sha256 === "string") {
+        url.searchParams.append("sha256", req.query.sha256);
+      } else {
+        console.warn("Missing sha256 query param");
+      }
+      const fetchRes = await fetch(url, {
+        headers: this.getAuthHeaders(data.token)
+      });
+      const contentType = fetchRes.headers.get("content-type");
+      if (contentType !== null) {
+        res.setHeader("Content-Type", contentType);
+      }
+      const contentLength = fetchRes.headers.get("content-length");
+      if (contentLength !== null) {
+        res.setHeader("Content-Length", contentLength);
+      }
+      if (fetchRes.ok) {
+        if (fetchRes.body) {
+          new BrowserReadableStreamWrapper(fetchRes.body).pipe(res);
+        } else {
+          console.warn("No body in response");
+          res.sendStatus(500);
+        }
+      } else {
+        res.sendStatus(fetchRes.status);
+      }
+    });
+  }
   async authorize(req, res) {
     const {
       redirect_to
@@ -1245,10 +1313,18 @@ class ProxyValServer {
         schema,
         source
       } = req.query;
+      const commit = this.options.gitCommit;
+      if (!commit) {
+        res.status(401).json({
+          error: "Could not detect the git commit. Check if env is missing VAL_GIT_COMMIT."
+        });
+        return;
+      }
       const params = new URLSearchParams({
         patch: (patch === "true").toString(),
         schema: (schema === "true").toString(),
-        source: (source === "true").toString()
+        source: (source === "true").toString(),
+        commit
       });
       const url = new URL(`/v1/tree/${this.options.valName}/heads/${this.options.gitBranch}/${req.params["0"]}/?${params}`, this.options.valContentUrl);
       const json = await fetch(url, {
@@ -1258,12 +1334,10 @@ class ProxyValServer {
     });
   }
   async postPatches(req, res) {
-    const {
-      commit
-    } = req.query;
-    if (typeof commit !== "string" || typeof commit === "undefined") {
+    const commit = this.options.gitCommit;
+    if (!commit) {
       res.status(401).json({
-        error: "Missing or invalid commit query param"
+        error: "Could not detect the git commit. Check if env is missing VAL_GIT_COMMIT."
       });
       return;
     }
@@ -1274,23 +1348,25 @@ class ProxyValServer {
       token
     }) => {
       // First validate that the body has the right structure
-      const patchJSON = PatchJSON.safeParse(req.body);
+      const patchJSON = z.z.record(PatchJSON).safeParse(req.body);
       if (!patchJSON.success) {
         res.status(401).json(patchJSON.error.issues);
         return;
       }
       // Then parse/validate
-      const patch$1 = patch.parsePatch(patchJSON.data);
-      if (fp.result.isErr(patch$1)) {
-        res.status(401).json(patch$1.error);
-        return;
-      }
-      const url = new URL(`/v1/tree/${this.options.valName}/heads/${this.options.gitBranch}/${req.params["0"]}/?${params}`, this.options.valContentUrl);
+      // TODO:
+      const patch = patchJSON.data;
+      // const patch = parsePatch(patchJSON.data);
+      // if (result.isErr(patch)) {
+      //   res.status(401).json(patch.error);
+      //   return;
+      // }
+      const url = new URL(`/v1/patches/${this.options.valName}/heads/${this.options.gitBranch}/${req.params["0"]}/?${params}`, this.options.valContentUrl);
       // Proxy patch to val.build
       const fetchRes = await fetch(url, {
         method: "POST",
         headers: this.getAuthHeaders(token, "application/json"),
-        body: JSON.stringify(patch$1)
+        body: JSON.stringify(patch)
       });
       if (fetchRes.ok) {
         res.status(fetchRes.status).json(await fetchRes.json());
@@ -1566,7 +1642,8 @@ class LocalValServer {
       const modules = Object.fromEntries(serializedModuleContent.map(serializedModuleContent => {
         const module = {
           schema: serializedModuleContent.schema,
-          source: serializedModuleContent.source
+          source: serializedModuleContent.source,
+          errors: serializedModuleContent.errors
         };
         return [serializedModuleContent.path, module];
       }));
@@ -1574,9 +1651,7 @@ class LocalValServer {
         modules,
         git: this.options.git
       };
-      return walk(rootDir).then(async () => {
-        res.send(JSON.stringify(apiTreeResponse));
-      });
+      res.send(JSON.stringify(apiTreeResponse));
     } catch (err) {
       console.error(err);
       res.sendStatus(500);
@@ -1589,24 +1664,23 @@ class LocalValServer {
     return disable(req, res);
   }
   async postPatches(req, res) {
-    var _getPathFromParams;
-    const id = (_getPathFromParams = getPathFromParams(req.params)) === null || _getPathFromParams === void 0 ? void 0 : _getPathFromParams.replace("/~", "");
-
     // First validate that the body has the right structure
-    const patchJSON = PatchJSON.safeParse(req.body);
-    console.log("patch id", id, patchJSON);
+    const patchJSON = z.z.record(PatchJSON).safeParse(req.body);
     if (!patchJSON.success) {
       res.status(401).json(patchJSON.error.issues);
       return;
     }
-    // Then parse/validate
-    const patch$1 = patch.parsePatch(patchJSON.data);
-    if (fp.result.isErr(patch$1)) {
-      res.status(401).json(patch$1.error);
-      return;
-    }
     try {
-      await this.options.service.patch(id, patch$1.value);
+      for (const moduleId in patchJSON.data) {
+        // Then parse/validate
+        // TODO: validate all and then fail instead:
+        const patch$1 = patch.parsePatch(patchJSON.data[moduleId]);
+        if (fp.result.isErr(patch$1)) {
+          res.status(401).json(patch$1.error);
+          return;
+        }
+        await this.options.service.patch(moduleId, patch$1.value);
+      }
       res.json({});
     } catch (err) {
       if (err instanceof patch.PatchError) {
@@ -1637,6 +1711,9 @@ class LocalValServer {
   logout(req, res) {
     return this.badRequest(req, res);
   }
+  getFiles(req, res) {
+    return this.badRequest(req, res);
+  }
 }
 
 async function _createRequestListener(route, opts) {