@server/next 0.16.0 → 0.17.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@server/next",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "An experimental reimplementation of server.js focused on the DX",
   "homepage": "https://node-server.com/",
   "repository": "https://github.com/franciscop/server-next.git",
@@ -9,6 +9,7 @@
   "author": "Francisco Presencia <public@francisco.io> (https://francisco.io/)",
   "license": "UNLICENSED",
   "scripts": {
+    "demo": "nodemon ./demo/app.js",
     "start": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch",
     "size": "echo \"$(gzip -c index.js | wc -c) bytes\" # Only for Unix",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js"

package/readme.md

@@ -7,29 +7,29 @@
 A fully-fledged web server for Node.js, with all the basics covered for you:
 
 ```js
-import server, { get, post, put, use } from 'server';
+import server, { get, post, put, use } from "server";
 
 // Create a running instance of the server
 const app = server(config, [pluginA, pluginB]);
 
 // Attach handlers to the instance
 app([
-  get('/users', getUsers),
-  post('/users', createUser),
-  put('/users/:id', editUser),
-  use('/admin/*', dashboard)
+  get("/users", getUsers),
+  post("/users", createUser),
+  put("/users/:id", editUser),
+  use("/admin/*", dashboard),
 ]);
 ```
 
-It includes all the things you would expect from a modern Server framework, like routing, static file serving, body+file parsing, gzip+brotli, streaming, server-timing, plugins\*, etc.
+It includes all the things you would expect from a modern Server framework, like routing, static file serving, options\*, body+file parsing, gzip+brotli, streaming, server-timing, plugins\*, etc.
 
 > \* not yet available
 
-## Upgradingn server
+## Upgrading server
 
 Why? The ecosystem is moving out of server-rendered websites so we are as well. Now instead we treat APIs as first-class citizens. Desired improvements (WIP!):
 
-- Tiny footprint with no dependencies, all bundled in a single file. Installing and using the full library takes under 10kb (target limit).
+- Tiny footprint with no dependencies\*, all bundled in a single file. Installing and using the full library takes under 10kb (target limit).
 - Faster! Reimplemented from scratch for speed. With raw ES6+ and a tiny code footprint, your server will fly.
 - Modern ES6+ESM syntax for both the library and examples.
 - Error handling improved greatly.
@@ -37,6 +37,13 @@ Why? The ecosystem is moving out of server-rendered websites so we are as well.
 - Changed the reply logic greatly, including the removal of `render()`. This is the main reason express was removed. Many servers don't need render() at all.
 - **[security]** Removed mandatory CSRF token, since this is only useful for server-rendered pages and not for SPA. You can activate it with a single option as before.
 
+Major changes:
+
+- New fully fledged `ctx.url` that extends [`URL`](https://developer.mozilla.org/en-US/docs/Web/API/URL) object inside `ctx` (also note: `ctx.url` is no longer a string):
+  - `ctx.params` is now `ctx.url.params`, e.g. `ctx.url.params.id`.
+  - `ctx.query` is now `ctx.url.query`, e.g. `ctx.url.params.search`.
+  - `ctx.path` is now `ctx.url.path` (or `ctx.url.pathname`).
+  - All URL properties are available, like `ctx.url.port`, `ctx.url.searchParams`, etc.
 
 ## Progress
 
@@ -48,7 +55,7 @@ Why? The ecosystem is moving out of server-rendered websites so we are as well.
   - A string and it'll be sent as plain text or html (if it starts with "<")
   - A readStream and it'll be piped to the response
   - An object with `status`, `body` and `headers` and it'll be set raw.
-
+- Response compression works
 
 ## Some plugins
 
@@ -85,3 +92,20 @@ app([
   })
 ]);
 ```
+
+## Examples
+
+### Streams
+
+Creating a 100x100px thumbnail on the fly with Sharp:
+
+```js
+// createThumbnail.js
+import { get } from "@server/next";
+import sharp from "sharp";
+
+export default function createThumbnail(ctx) {
+  // Return a pipe, which will be streamed to the output
+  return sharp(ctx.url.params.name).resize(100, 100, { fit: "cover" }).png();
+}
+```

package/src/RequestLogger.js

@@ -1,3 +1,4 @@
+import path from "node:path";
 import Spinnies from "spinnies";
 import color from "./color.js";
 
@@ -17,6 +18,15 @@ const format = (n, [below, above], limit = Infinity) => {
   }
 };
 
+const range = (char, num) => {
+  const cols = process.stdout.columns || 80;
+  let str = "";
+  for (let i = 0; i < cols; i += char.length) {
+    str += char;
+  }
+  return str;
+};
+
 function simpleType(type) {
   const simpler = {
     "text/html": "html",
@@ -24,6 +34,7 @@ function simpleType(type) {
     "image/svg+xml": "svg",
     "image/png": "png",
     "text/css": "css",
+    "text/javascript": "js",
     "application/javascript": "js",
     "application/json": "json",
     "text/markdown": "md",
@@ -31,6 +42,8 @@ function simpleType(type) {
   return simpler[type] || type;
 }
 
+const cwd = path.resolve("./");
+
 let spinnies;
 export default function RequestLogger(ctx) {
   this.id = Math.round(Math.random() * 100000);
@@ -45,26 +58,66 @@ export default function RequestLogger(ctx) {
   }
 
   this.end = function (ctx) {
-    if (!isProduction) {
-      const status = ctx.res.status;
+    if (isProduction) return;
+    const status = ctx.res.status;
 
-      const paddedPath = `${ctx.url?.path || ctx.url} {dim}`.padEnd(30, "╌");
-      const statColor =
-        status < 300 ? "green" : status < 500 ? "yellow" : "red";
-      const statusBlock = `{${statColor}}[${ctx.res.status}]{/}`;
-      const resSize = format(ctx.res.size || 0, ["b", "kb"], 100000);
-      const t = Math.round(ctx.time._total - ctx.time._init);
-      const resTime = format(t, ["ms", "s"], 1000);
+    const paddedPath = `${ctx.url?.path || ctx.url} {dim}`.padEnd(30, "─");
+    const statColor = status < 300 ? "green" : status < 500 ? "yellow" : "red";
+    const statusBlock = `{${statColor}}[${ctx.res.status}]{/}`;
+    const resSize = format(ctx.res.size || 0, ["b", "kb"], 100000);
+    const t = Math.round(ctx.time._total - ctx.time._init);
+    const resTime = format(t, ["ms", "s"], 1000);
 
-      const type = simpleType(
-        ctx.res.type || ctx.res.headers["content-type"] || "----"
-      );
-      const method = ("[" + ctx.method + "]").padEnd(6, " ");
-      const reqText = `{dim}${method}{/} ${paddedPath}`;
-      const resText = `${statusBlock} ${resSize} ${resTime} ${type}`;
-      const text = color(`${reqText}╌›{/} ${resText}`);
+    const type = simpleType(
+      ctx.res.type || ctx.res.headers["content-type"] || "----"
+    );
+    const method = ("[" + ctx.method + "]").padEnd(6, " ");
+    const reqText = `{dim}${method}{/} ${paddedPath}`;
+    const resText = `${statusBlock} ${resSize} ${resTime} ${type}`;
+    const text = color(`${reqText}─›{/} ${resText}`);
 
+    this.text = text;
+    if (statColor !== "red") {
       spinnies.succeed(`spinner-${this.id}`, { text });
+    } else {
+      spinnies.fail(`spinner-${this.id}`, { text });
     }
   };
+
+  this.error = function (err) {
+    const cols = process.stdout.columns || 80;
+    console.log(color("{red}  ┌───────┬" + range("─").slice(12) + "┐{/}"));
+    console.log(
+      color(
+        `{red}  │ {bright}{red}Error{/} {red}│{/} ${err.message.padEnd(
+          cols - 14,
+          " "
+        )} {red}│{/}`
+      )
+    );
+    console.log(color("{red}  ├───────┼" + range("─").slice(12) + "┤{/}"));
+    console.log(
+      color(
+        err.stack
+          .split("\n")
+          .slice(1)
+          .map((line, i) => {
+            line = line.replace(/\s*at\s/, "");
+            line = line.replace("file://" + cwd, "{dim}$PWD{/}");
+            return (
+              `{red}  │{/} ${i === 0 ? "Trace" : "     "} {red}│{/} ` +
+              line.padEnd(cols - 6, " ") +
+              " {red}│{/}"
+            );
+          })
+          .join("\n")
+      )
+    );
+    console.log(color("{red}  └───────┴" + range("─").slice(12) + "┘{/}"));
+  };
+
+  this.reprint = function () {
+    if (isProduction) return;
+    console.log(color(`{red}✖{/} `) + this.text);
+  };
 }

package/src/bucket.js

@@ -0,0 +1,28 @@
+import path from "node:path";
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+
+// A fake tiny implementation of a generic bucket, it needs
+// at the very least a read(id) and write(id, value), both returning
+// promises. If possible both are also pipeable/streamable.
+export default function (root) {
+  const absolute = (name) => {
+    if (!name) throw new Error(`File name is required`);
+    return path.resolve(path.join(root, name));
+  };
+
+  return {
+    read: (name, type = "utf8") => {
+      const fullPath = absolute(name);
+      return fsp.readFile(fullPath, type);
+    },
+    write: (name, value, type = "utf8") => {
+      const fullPath = absolute(name);
+      if (value) {
+        return fsp.writeFile(fullPath, value, type).then(() => fullPath);
+      } else {
+        return fs.createWriteStream(fullPath);
+      }
+    },
+  };
+}

package/src/color.js

@@ -12,7 +12,11 @@ const map = {
   bgblue: 44, bgmagenta: 45, bgcyan: 46, bgwhite: 47,
 };
 
-const replace = (k) => (process.env.NO_COLOR ? "" : `\x1b[${map[k]}m`);
+const replace = (k) => {
+  if (process.env.NO_COLOR) return "";
+  if (!(k in map)) throw new Error(`"{${k}}" is not a valid color`);
+  return `\x1b[${map[k]}m`;
+};
 
 export default function color(str, ...vals) {
   if (typeof str === "string") {
@@ -21,6 +25,6 @@ export default function color(str, ...vals) {
       .replaceAll(/\{\/\w*\}/g, replace("reset"));
   }
 
-  // For template literals, put them together first and then color it
+  // Template literals, put them together first and then color them
   return color(str[0] + vals.map((v, i) => v + str[i + 1]).join(""));
 }

package/src/getMime.js

@@ -0,0 +1,48 @@
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import { spawn } from "node:child_process";
+
+// These are some mime types that are not properly produced by
+// the file --mime-type command, and can be inferred by their
+// filename extension easily
+// Note: text/javascript is the recommended now https://stackoverflow.com/a/876805/938236
+const extensions = {
+  css: "text/css",
+  js: "text/javascript",
+};
+
+const exists = (file) => {
+  return fsp.stat(file, fs.constants.F_OK).then(
+    (stat) => stat.isFile(),
+    () => false
+  );
+};
+
+function cmd(...args) {
+  const { stdout: stream } = spawn(...args);
+  const chunks = [];
+  return new Promise((resolve, reject) => {
+    stream.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
+    stream.on("error", (err) => reject(err));
+    stream.on("end", () =>
+      resolve(Buffer.concat(chunks).toString("utf8").trim())
+    );
+  });
+}
+
+export default async function (file) {
+  const cwd = await fsp.realpath("./");
+  file = await fsp.realpath(file);
+
+  const ext = file.split(".").pop();
+  if (extensions[ext]) return extensions[ext];
+
+  // It _is_ a file
+  if (!(await exists(file))) return false;
+  // Inside the CWD directory
+  if (!file.startsWith(cwd)) return false;
+
+  // The two conditions above give us enough confidence to run
+  // this, but even then we use spawn() to avoid some nasty injections
+  return cmd("file", ["--mime-type", "-b", file]);
+}

package/src/index.js

@@ -1,3 +1,5 @@
+import "dotenv/config";
+
 import http from "node:http";
 import fs from "node:fs";
 import fsp from "node:fs/promises";
@@ -6,14 +8,13 @@ import zlib from "node:zlib";
 import { pipeline } from "node:stream/promises";
 import { Readable, PassThrough } from "node:stream";
 
-import "dotenv/config";
-
 import ServerUrl from "./ServerUrl.js";
 import RequestLogger from "./RequestLogger.js";
 
+import logger from "./logger.js";
+import getMime from "./getMime.js";
 import pathPattern from "./pathPattern.js";
 import parseBody from "./parseBody.js";
-import color from "./color.js";
 
 const isProduction = process.env.NODE_ENV === "production";
 
@@ -24,13 +25,6 @@ const getIp = (req) =>
   req.socket.remoteAddress ||
   req.connection.socket.remoteAddress;
 
-const exists = (file) => {
-  return fsp.stat(file, fs.constants.F_OK).then(
-    (stat) => stat.isFile(),
-    () => false
-  );
-};
-
 const measure = (ctx) => {
   const sizeUp = new PassThrough();
   ctx.res.size = 0;
@@ -83,19 +77,32 @@ const getCtx = (req) => ({
 const createApp = (server) => {
   const app = function (...mid) {
     app.middleware.push(...mid.flat());
+
+    // Final error handling
+    app.middleware.push({
+      handle: (error) => {
+        // console.clear();
+        // console.log("ERROR!", error);
+        return {
+          status: 500,
+          body: JSON.stringify({ error: error.message }),
+          type: "application/json",
+        };
+      },
+    });
   };
 
+  // Static middleware
   app.middleware = [
     async function (ctx) {
-      if (ctx.method === "get") {
-        const file = path.join(process.cwd(), ctx.options.public, ctx.url.path);
-        const size = await fsp.stat(file).then(
-          (stat) => stat.isFile() && stat.size,
-          (err) => false
-        );
-        if (size) {
-          return fs.createReadStream(file);
-        }
+      if (ctx.method !== "get") return;
+      const file = path.join(process.cwd(), ctx.options.public, ctx.url.path);
+      const size = await fsp.stat(file).then(
+        (stat) => stat.isFile() && stat.size,
+        () => false
+      );
+      if (size) {
+        return fs.createReadStream(file);
       }
     },
   ];
@@ -111,12 +118,49 @@ const createApp = (server) => {
   return app;
 };
 
-const logStart = (port) => {
-  if (isProduction) return;
-  const routes = Object.values(api).flat().length;
-  console.log(
-    color(`Running on {under}http://localhost:${port}/{/} (${routes} routes)`)
-  );
+let ttyWarn = null;
+// if (!process.stdin.isTTY) {
+//   ttyWarn = "Invalid terminal; cannot accept user input";
+// }
+// if (ttyWarn && process.env._.endsWith("nodemon")) {
+//   ttyWarn = "when running with nodemon, please pass the flag --no-stdin";
+// }
+
+const parseOutput = async (res, data) => {
+  // Plain number
+  if (typeof data === "number") {
+    res.status = data;
+    res.body = "";
+    res.type = "text/plain";
+    return res;
+  }
+
+  // Plain string, which can only be text or html
+  if (typeof data === "string") {
+    res.status = 200;
+    res.body = data;
+    const isHtml = data.trim().startsWith("<");
+    res.type = isHtml ? "text/html" : "text/plain";
+    return res;
+  }
+
+  // When piping the output
+  if (data.pipe) {
+    res.body = data;
+    res.status = 200;
+    // It's a file; find its mimetype
+    if (res.body.path) {
+      res.type = await getMime(res.body.path);
+    }
+    return res;
+  }
+
+  // Plain object, merge it with the existing one
+  if (data.type) res.type = data.type;
+  res.headers = { ...res.headers, ...(data.headers || {}) };
+  res.body = data.body || ""; // This could also be a pipe and that's okay
+  res.status = data.status || 200;
+  return res;
 };
 
 export default function (options = {}, plugins) {
@@ -126,46 +170,43 @@ export default function (options = {}, plugins) {
 
   const server = http.createServer(async (req, res) => {
     const logger = new RequestLogger(req);
-    const ctx = { ...getCtx(req), options };
+    const ctx = { ...getCtx(req), bucket: options.bucket, options };
 
-    const parsed = await parseBody(ctx.req, ctx.headers["content-type"]);
+    const parsed = await parseBody(
+      ctx.req,
+      ctx.headers["content-type"],
+      ctx.bucket
+    );
     if (parsed) {
       ctx.body = parsed.body;
       ctx.files = parsed.files;
     }
 
     let out;
+    let err;
     ctx.res = { headers: {} };
     for (let cb of app.middleware) {
-      out = await cb(ctx);
-      if (out) {
-        if (typeof out === "number") {
-          // Plain number
-          ctx.res.status = out;
-          ctx.res.body = "";
-          ctx.res.headers["content-type"] = "text/plain";
-        } else if (typeof out === "string") {
-          // Plain string
-          ctx.res.status = 200;
-          ctx.res.body = out;
-          const isHtml = out.trim().startsWith("<");
-          ctx.res.headers["content-type"] = isHtml ? "text/html" : "text/plain";
-        } else {
-          if (out.pipe) {
-            ctx.res.body = out;
-            ctx.res.status = 200;
-          } else {
-            // Plain object
-            if (out.type) ctx.res.headers["content-type"] = out.type;
-            ctx.res.headers = { ...ctx.res.headers, ...(out.headers || {}) };
-            ctx.res.body = out.body || "";
-            ctx.res.status = out.status || 200;
+      try {
+        if (err) {
+          if (cb.handle) {
+            out = await cb.handle(err);
           }
+        } else {
+          out = await cb(ctx);
         }
-        break;
+        if (out) {
+          ctx.res = await parseOutput(ctx.res, out);
+          break;
+        }
+      } catch (error) {
+        err = error;
       }
     }
 
+    if (ctx.res.type) {
+      ctx.res.headers["content-type"] = ctx.res.type;
+    }
+
     ctx.time._total = performance.now();
     ctx.res.headers["server-timing"] = ctx.res.headers["server-timing"] || "";
     Object.entries(ctx.time).forEach(([name, value], i, times) => {
@@ -189,10 +230,6 @@ export default function (options = {}, plugins) {
       ctx.res.body = Readable.from([ctx.res.body]);
     }
 
-    if (ctx.res.body.path) {
-      ctx.res.type = ctx.res.body.path.split(".").pop();
-    }
-
     if (compress) {
       await pipeline(ctx.res.body, compress(), measure(ctx), res);
     } else {
@@ -213,6 +250,9 @@ export default function (options = {}, plugins) {
     );
 
     logger.end(ctx);
+    if (err) {
+      logger.error(err);
+    }
   });
 
   const app = createApp(server);
@@ -226,7 +266,7 @@ export default function (options = {}, plugins) {
       }
     } else {
       app.events.ready.forEach((cb) => cb({ options }));
-      logStart(options.port);
+      logger.header({ api, options });
     }
   });
 

package/src/logger.js

@@ -0,0 +1,21 @@
+import color from "./color.js";
+
+const isProduction = process.env.NODE_ENV === "production";
+
+const header = function ({ api, options }) {
+  if (isProduction) return;
+  const routes = Object.values(api).flat().length;
+  console.clear();
+  console.log(
+    color(
+      `{green}Started server successfully{/} (${routes} routes). See app in the browser:
+
+🔗 {under}http://localhost:${options.port}/{/}
+
+{dim}Press [ctrl+c] to exit the server.{/}
+`
+    )
+  );
+};
+
+export default { header };

package/src/parseBody.js

@@ -23,17 +23,32 @@ function getMatching(string, regex) {
 
 const getBody = async (req) => {
   return await new Promise((done) => {
-    let data = "";
+    const buffers = [];
     req.on("data", (chunk) => {
-      data += chunk;
+      buffers.push(chunk);
     });
     req.on("end", () => {
-      done(data);
+      done(Buffer.concat(buffers).toString("binary"));
     });
   });
 };
 
-export default async function Parse(req, contentType) {
+const nanoid = (size = 12) => {
+  let str = "";
+  while (str.length < size + 2) {
+    str += Math.round(Math.random() * 1000000).toString(16);
+  }
+  return str.slice(0, size);
+};
+
+const saveFile = async (name, value, bucket) => {
+  const ext = name.split(".").pop();
+  const id = `file-${nanoid(12)}.${ext}`;
+  await bucket.write(id, value, "binary");
+  return id;
+};
+
+export default async function Parse(req, contentType, bucket) {
   const rawData = await (typeof req === "string" ? req : getBody(req));
   if (!rawData) return null;
 
@@ -44,10 +59,7 @@ export default async function Parse(req, contentType) {
   const boundary = getBoundary(contentType);
   if (!boundary) return null;
 
-  let result = {};
-
   const body = {};
-  const files = {};
 
   const rawDataArray = rawData.split(boundary);
   for (let item of rawDataArray) {
@@ -56,41 +68,26 @@ export default async function Parse(req, contentType) {
       .trim()
       .replace(/\[\]$/, "");
     if (!name) continue;
-    const value = getMatching(item, /(?:\r\n\r\n)([\S\s]*)(?:\r\n--$)/);
+
+    let value = getMatching(item, /(?:\r\n\r\n)([\S\s]*)(?:\r\n--$)/);
     if (!value) continue;
 
+    // Check whether we have a filename. If we do, assign it to the value
     const filename = getMatching(item, /(?:filename=")(.*?)(?:")/).trim();
-    // It's a file!
     if (filename) {
-      const file = { name: filename };
-      const type = getMatching(item, /(?:Content-Type:)(.*?)(?:\r\n)/).trim();
-      if (type) {
-        file.type = type;
-      }
-      file.value = value;
+      value = await saveFile(filename, value, bucket);
+    }
 
-      // Already exists, so (maybe convert to an array) and push the item in it
-      if (files[name]) {
-        if (!Array.isArray(files.name)) {
-          files[name] = [files[name]];
-        }
-        files[name].push(file);
-      } else {
-        files[name] = file;
+    // Save the key-value, accounting for possibly repeated keys
+    if (body[name]) {
+      if (!Array.isArray(body[name])) {
+        body[name] = [body[name]];
       }
-
-      // It's a body
+      body[name].push(value);
     } else {
-      if (body[name]) {
-        if (!Array.isArray(body[name])) {
-          body[name] = [body[name]];
-        }
-        body[name].push(value);
-      } else {
-        body[name] = value;
-      }
+      body[name] = value;
     }
   }
 
-  return { body, files };
+  return body;
 }

package/src/parseBody.test.js

@@ -36,34 +36,23 @@ const getBody = () => {
   return body;
 };
 
+const matchMd = expect.stringMatching(/^file-\w{12}.md$/);
+const matchTxt = expect.stringMatching(/^file-\w{12}.txt$/);
+
 describe("parseBody", () => {
   it("can parse the example body", async () => {
-    const parsed = await parseBody(
+    const body = await parseBody(
       getBody(),
-      "multipart/form-data; boundary=----WebKitFormBoundaryvef1fLxmoUdYZWXp"
+      "multipart/form-data; boundary=----WebKitFormBoundaryvef1fLxmoUdYZWXp",
+      { write: (id) => id }
     );
-    expect(parsed.body).toEqual({
+    expect(body).toMatchObject({
       hello: "world",
       test: ["test message 123456", "test message number two"],
     });
-    expect(parsed.files).toEqual({
-      profile: {
-        name: "profile.md",
-        type: "text/plain",
-        value: "@11X111Y\r\n111Z\rCCCC\nCCCC\r\nCCCCC@\r\n",
-      },
-      gallery: [
-        {
-          name: "A.txt",
-          type: "text/plain",
-          value: "@11X111Y\r\n111Z\rCCCC\nCCCC\r\nCCCCC@\r\n",
-        },
-        {
-          name: "C.txt",
-          type: "text/plain",
-          value: "@CCCCCCY\r\nCCCZ\rCCCW\nCCC0\r\n666@",
-        },
-      ],
+    expect(body).toMatchObject({
+      profile: matchMd,
+      gallery: [matchTxt, matchTxt],
     });
   });
 });