@teleporthq/teleport-plugin-next-workflows 0.44.11 → 0.45.0

package/src/runtime-storage-generator.ts

@@ -1,5 +1,6 @@
 import { generateCommonJsSessionTokenResolverCode } from './session-cookie-resolver'
 import { ROLE_OF_TOKEN_CODE } from './workflow-auth-generator'
+import { securityRuntimeSpecifier } from './security-runtime'
 
 const FILE_STORAGE_NODE_TYPES = new Set([
   'file-storage-upload',
@@ -71,7 +72,51 @@ export const VISITOR_UPLOAD_MIME_TYPES: ReadonlyArray<string> = [
 export const VISITOR_MAX_FILE_BYTES = 5 * 1024 * 1024
 export const VISITOR_MAX_FILES = 10
 
-export const generateRuntimeStorageUploadRoute = (): string => {
+/**
+ * How much one visitor (by address, IPv6 by its /64) and the whole site may
+ * upload, through the shared security runtime:
+ * - VISITOR_UPLOAD_LIMITS: requests, `[limit, windowSeconds]`, checked before
+ *   the body is read;
+ * - VISITOR_UPLOAD_MB_PER_DAY: bytes, counted per file in whole MB (a file
+ *   counts at least 1), after the form passed its checks;
+ * - SITE_VISITOR_FILES_PER_DAY: files from all visitors together, counted only
+ *   for forms that passed, so junk requests cannot pause everyone's uploads.
+ * The storage worker also holds all guests together to their share of the
+ * project's quota. Without these a guest could fill the merchant's quota with
+ * allowed files until the admin's own uploads fail.
+ */
+export const VISITOR_UPLOAD_LIMITS: ReadonlyArray<[number, number]> = [
+  [20, 3600],
+  [60, 86400],
+]
+export const VISITOR_UPLOAD_MB_PER_DAY = 25
+export const SITE_VISITOR_FILES_PER_DAY = 500
+
+/** The true values of the `private` flag, read the way the storage worker reads them. */
+export const PRIVATE_FLAG_VALUES: ReadonlyArray<string> = ['true', '1', 'yes']
+
+/** The extensions a proven type keeps; any other is replaced by the first. */
+export const PROVEN_TYPE_EXTENSIONS: Record<string, string[]> = {
+  'image/jpeg': ['.jpg', '.jpeg'],
+  'image/png': ['.png'],
+  'image/webp': ['.webp'],
+  'image/gif': ['.gif'],
+  'application/pdf': ['.pdf'],
+}
+
+/**
+ * Tells the storage worker which policy a forwarded upload gets. Only this
+ * route sends it (the request carries the project key): `visitor` uploads
+ * meet the worker's own allow-list as well as this route's, `trusted` ones
+ * (the store's server code, an admin) upload any type as before. A site
+ * exported before this header existed sends none, and the worker refuses
+ * active content (HTML, SVG, XML, scripts) from it.
+ */
+export const UPLOAD_ACCESS_HEADER = 'x-teleport-upload-access'
+
+export const generateRuntimeStorageUploadRoute = (
+  options: { withSessionRevocation?: boolean } = {}
+): string => {
   return `var RUNTIME_STORAGE_URL = process.env.RUNTIME_STORAGE_URL;
 var RUNTIME_STORAGE_API_KEY = process.env.RUNTIME_STORAGE_API_KEY;
 var RUNTIME_STORAGE_PROJECT_ID = process.env.RUNTIME_STORAGE_PROJECT_ID;
@@ -80,7 +125,9 @@ var RUNTIME_STORAGE_PROJECT_ID = process.env.RUNTIME_STORAGE_PROJECT_ID;
 // must become a 504, never a request the browser waits on forever.
 var UPLOAD_TIMEOUT_MS = ${UPLOAD_PROXY_TIMEOUT_MS};
 
-${generateCommonJsSessionTokenResolverCode()}
+${generateCommonJsSessionTokenResolverCode('../../..', {
+  withRevocation: options.withSessionRevocation !== false,
+})}
 ${ROLE_OF_TOKEN_CODE}
 
 // What a visitor may upload — see TRUSTED_UPLOAD_ROLES in the generator.
@@ -94,6 +141,14 @@ var VISITOR_MAX_REQUEST_BYTES = VISITOR_MAX_FILES * VISITOR_MAX_FILE_BYTES + 256
 var CRLF = Buffer.from([13, 10]);
 var BLANK_LINE = Buffer.from([13, 10, 13, 10]);
 
+var __security = require('${securityRuntimeSpecifier('../../..')}');
+var VISITOR_UPLOAD_LIMITS = ${JSON.stringify(VISITOR_UPLOAD_LIMITS)};
+var VISITOR_UPLOAD_MB_PER_DAY = ${VISITOR_UPLOAD_MB_PER_DAY};
+var SITE_VISITOR_FILES_PER_DAY = ${SITE_VISITOR_FILES_PER_DAY};
+var PRIVATE_FLAG_VALUES = ${JSON.stringify(PRIVATE_FLAG_VALUES)};
+var PROVEN_TYPE_EXTENSIONS = ${JSON.stringify(PROVEN_TYPE_EXTENSIONS)};
+var UPLOAD_ACCESS_HEADER = '${UPLOAD_ACCESS_HEADER}';
+
 // Only server code can read NEXTAUTH_SECRET, so a browser cannot present it.
 function presentsAppSecret(req) {
   var given = req && req.headers ? req.headers['x-internal-data-secret'] : '';
@@ -156,23 +211,113 @@ function multipartBoundary(contentType) {
   return match ? (match[1] || match[2] || '').trim() : '';
 }
 
+// RFC 7230 token characters: what busboy accepts as a disposition type or a
+// parameter name.
+var TOKEN_CHAR = /[!#$%&'*+.^_\\x60|~0-9A-Za-z-]/;
+
+// A Content-Disposition value read the way busboy (the storage worker's
+// parser) reads it: the type, then ';'-separated parameters whose value is a
+// token, a quoted string (backslash escapes) or, for a name ending in '*', an
+// RFC 5987 extended value (charset'language'percent-encoded). null when it is
+// malformed. Deciding "is this a file" on any other reading lets a part the
+// worker stores as a file pass this route as a plain field.
+function parseDisposition(value) {
+  var i = 0;
+  while (i < value.length && TOKEN_CHAR.test(value[i])) i++;
+  var type = value.slice(0, i).toLowerCase();
+  if (!type) return null;
+  var params = {};
+  while (i < value.length) {
+    while (value[i] === ' ' || value[i] === '\\t') i++;
+    if (i >= value.length) break;
+    if (value[i] !== ';') return null;
+    i++;
+    while (value[i] === ' ' || value[i] === '\\t') i++;
+    var nameStart = i;
+    while (i < value.length && TOKEN_CHAR.test(value[i])) i++;
+    if (i === nameStart || value[i] !== '=') return null;
+    var name = value.slice(nameStart, i).toLowerCase();
+    i++;
+    var paramValue = '';
+    if (name.charAt(name.length - 1) === '*') {
+      var ext = /^([^']*)'[^']*'([^;\\s]*)/.exec(value.slice(i));
+      if (!ext) return null;
+      i += ext[0].length;
+      try {
+        paramValue = /^utf-?8$/i.test(ext[1])
+          ? decodeURIComponent(ext[2])
+          : unescape(ext[2]);
+      } catch (e) {
+        return null;
+      }
+    } else if (value[i] === '"') {
+      i++;
+      var closed = false;
+      while (i < value.length) {
+        var ch = value[i];
+        if (ch === '\\\\' && i + 1 < value.length) {
+          paramValue += value[i + 1];
+          i += 2;
+          continue;
+        }
+        if (ch === '"') {
+          closed = true;
+          i++;
+          break;
+        }
+        paramValue += ch;
+        i++;
+      }
+      if (!closed) return null;
+    } else {
+      var valueStart = i;
+      while (i < value.length && TOKEN_CHAR.test(value[i])) i++;
+      if (i === valueStart) return null;
+      paramValue = value.slice(valueStart, i);
+    }
+    if (!Object.prototype.hasOwnProperty.call(params, name)) params[name] = paramValue;
+  }
+  return { type: type, params: params };
+}
+
+// One part's name, filename and declared type. A file is what the storage
+// worker stores as one: a part whose 'filename*' or 'filename' is not empty
+// ('filename*' first), quoted, unquoted or extended alike. The ambiguous flag marks a
+// part two parsers could read differently (a second disposition, a folded
+// header line, a disposition that is not well-formed form-data); a visitor's
+// form with one is refused.
 function readPart(headerText, data) {
-  var part = { name: '', filename: null, type: '', data: data };
+  var part = { name: '', filename: null, type: '', data: data, ambiguous: false };
+  var sawDisposition = false;
   var lines = headerText.split(String.fromCharCode(13, 10));
   for (var i = 0; i < lines.length; i++) {
+    if (lines[i].charAt(0) === ' ' || lines[i].charAt(0) === '\\t') {
+      part.ambiguous = true;
+      continue;
+    }
     var colon = lines[i].indexOf(':');
     if (colon <= 0) continue;
     var key = lines[i].slice(0, colon).trim().toLowerCase();
     var value = lines[i].slice(colon + 1).trim();
     if (key === 'content-disposition') {
-      var name = /[; ]name="([^"]*)"/i.exec(value);
-      var filename = /filename="([^"]*)"/i.exec(value);
-      part.name = name ? name[1] : '';
-      part.filename = filename ? filename[1] : null;
+      if (sawDisposition) {
+        part.ambiguous = true;
+        continue;
+      }
+      sawDisposition = true;
+      var disposition = parseDisposition(value);
+      if (!disposition || disposition.type !== 'form-data') {
+        part.ambiguous = true;
+        continue;
+      }
+      part.name = disposition.params.name || '';
+      var filename = disposition.params['filename*'] || disposition.params.filename || '';
+      part.filename = filename ? filename : null;
     } else if (key === 'content-type') {
       part.type = value.split(';')[0].trim().toLowerCase();
     }
   }
+  if (!sawDisposition) part.ambiguous = true;
   return part;
 }
 
@@ -216,8 +361,12 @@ function visitorUploadRefusal(parts) {
   var files = 0;
   for (var i = 0; i < parts.length; i++) {
     var part = parts[i];
+    if (part.ambiguous) {
+      return { status: 400, code: 'UPLOAD_INVALID', message: 'The upload is not a file form.' };
+    }
     if (part.filename === null) {
-      if (part.name === 'private' && String(part.data.toString('utf8')).trim() === 'true') {
+      // Every value the storage worker takes as private (true, 1, yes; any case).
+      if (part.name === 'private' && PRIVATE_FLAG_VALUES.indexOf(part.data.toString('utf8').trim().toLowerCase()) !== -1) {
         return { status: 403, code: 'UPLOAD_PRIVATE_FORBIDDEN', message: 'Only the store can upload private files.' };
       }
       continue;
@@ -239,6 +388,103 @@ function visitorUploadRefusal(parts) {
   return null;
 }
 
+// The file's base name with the extension its proven type has: x.html holding
+// PNG bytes is stored as x.png, never under a name that says it is a page.
+function provenFileName(filename, type) {
+  var base = String(filename).split(/[\\\\/]/).pop() || 'file';
+  var allowed = PROVEN_TYPE_EXTENSIONS[type] || [];
+  var dot = base.lastIndexOf('.');
+  var ext = dot > 0 ? base.slice(dot).toLowerCase() : '';
+  if (allowed.indexOf(ext) !== -1) return base;
+  var stem = dot > 0 ? base.slice(0, dot) : base;
+  var stemDot = stem.lastIndexOf('.');
+  if (stemDot > 0 && allowed.indexOf(stem.slice(stemDot).toLowerCase()) !== -1) return stem;
+  return (stem || 'file') + (allowed[0] || '');
+}
+
+// Header-safe: no quote, backslash or control character survives.
+function headerSafe(text) {
+  return String(text).replace(/["\\\\\\u0000-\\u001f\\u007f]/g, '_');
+}
+
+// The checked form, written afresh: one quoted disposition per part, each
+// file with the type its bytes proved and its base name, under a boundary of
+// our own. The storage worker parses this and nothing else, so what it stores
+// is exactly what visitorUploadRefusal passed — not a second reading of the
+// visitor's bytes. null when the boundary cannot be made unique.
+function serializeVisitorForm(parts) {
+  for (var attempt = 0; attempt < 3; attempt++) {
+    var boundary = '----tqupload' + require('crypto').randomBytes(16).toString('hex');
+    var marker = Buffer.from('--' + boundary);
+    var clash = false;
+    var chunks = [];
+    for (var i = 0; i < parts.length && !clash; i++) {
+      var part = parts[i];
+      if (part.data.indexOf(marker) !== -1) {
+        clash = true;
+        break;
+      }
+      var head = '--' + boundary + '\\r\\nContent-Disposition: form-data; name="' + headerSafe(part.name) + '"';
+      if (part.filename !== null) {
+        var type = sniffMimeType(part.data);
+        head += '; filename="' + headerSafe(provenFileName(part.filename, type)) + '"\\r\\nContent-Type: ' + type;
+      }
+      chunks.push(Buffer.from(head + '\\r\\n\\r\\n', 'utf8'), part.data, CRLF);
+    }
+    if (clash) continue;
+    chunks.push(Buffer.from('--' + boundary + '--\\r\\n'));
+    return { boundary: boundary, body: Buffer.concat(chunks) };
+  }
+  return null;
+}
+
+// A visitor's request allowance, before the body is read.
+// false = the 429 is already sent.
+async function visitorMayUpload(req, res) {
+  for (var i = 0; i < VISITOR_UPLOAD_LIMITS.length; i++) {
+    var ok = await __security.rateLimitRequest(req, res, {
+      scope: 'runtime-storage-upload',
+      limit: VISITOR_UPLOAD_LIMITS[i][0],
+      windowSeconds: VISITOR_UPLOAD_LIMITS[i][1],
+    });
+    if (!ok) return false;
+  }
+  return true;
+}
+
+// A checked form's cost, per file: its size in whole MB out of the visitor's
+// daily byte budget, then one file out of the site's daily visitor files.
+// false = the 429 is already sent.
+async function visitorMayStore(req, res, parts) {
+  var files = 0;
+  for (var i = 0; i < parts.length; i++) {
+    if (parts[i].filename === null) continue;
+    files += 1;
+    var units = Math.max(1, Math.ceil(parts[i].data.length / 1048576));
+    for (var u = 0; u < units; u++) {
+      var ok = await __security.rateLimitRequest(req, res, {
+        scope: 'runtime-storage-upload-mb',
+        limit: VISITOR_UPLOAD_MB_PER_DAY,
+        windowSeconds: 86400,
+      });
+      if (!ok) return false;
+    }
+  }
+  for (var f = 0; f < files; f++) {
+    var site = await __security.rateLimit({
+      key: 'site:visitor-uploads',
+      limit: SITE_VISITOR_FILES_PER_DAY,
+      windowSeconds: 86400,
+    });
+    if (!site.allowed) {
+      res.setHeader('Retry-After', String(site.retryAfter));
+      refuse(res, 429, 'UPLOAD_LIMIT_REACHED', 'Uploads are paused for now. Please try again later.');
+      return false;
+    }
+  }
+  return true;
+}
+
 module.exports = async function handler(req, res) {
   if (req.method !== 'POST') {
     res.status(405).json({ error: 'Method not allowed' });
@@ -253,8 +499,10 @@ module.exports = async function handler(req, res) {
   var trusted = await isTrustedUploader(req);
   var body = req;
   var contentLength = req.headers['content-length'];
+  var contentType = req.headers['content-type'];
 
   if (!trusted) {
+    if (!(await visitorMayUpload(req, res))) return;
     var declaredLength = parseInt(contentLength, 10);
     if (!isNaN(declaredLength) && declaredLength > VISITOR_MAX_REQUEST_BYTES) {
       refuse(res, 413, 'UPLOAD_TOO_LARGE', 'The upload is too large.');
@@ -285,6 +533,14 @@ module.exports = async function handler(req, res) {
       refuse(res, refusal.status, refusal.code, refusal.message);
       return;
     }
+    if (!(await visitorMayStore(req, res, parts))) return;
+    var checked = serializeVisitorForm(parts);
+    if (!checked) {
+      refuse(res, 400, 'UPLOAD_INVALID', 'The upload could not be read.');
+      return;
+    }
+    body = checked.body;
+    contentType = 'multipart/form-data; boundary=' + checked.boundary;
     contentLength = String(body.length);
   }
 
@@ -301,9 +557,12 @@ module.exports = async function handler(req, res) {
     var headers = {
       'Authorization': 'Bearer ' + RUNTIME_STORAGE_API_KEY,
     };
+    // Which of the storage worker's policies this upload gets (only this
+    // server can send it: the request carries the project key).
+    headers[UPLOAD_ACCESS_HEADER] = trusted ? 'trusted' : 'visitor';
 
-    if (req.headers['content-type']) {
-      headers['content-type'] = req.headers['content-type'];
+    if (contentType) {
+      headers['content-type'] = contentType;
     }
     if (contentLength) {
       headers['content-length'] = contentLength;
@@ -313,7 +572,7 @@ module.exports = async function handler(req, res) {
       method: 'POST',
       headers: headers,
       // A trusted caller's form is streamed through untouched; a visitor's was
-      // read in full to be checked, and is forwarded byte for byte.
+      // read in full, checked, and is forwarded as the form it was checked as.
       body: body,
       duplex: 'half',
       signal: controller.signal,

package/src/security-runtime/index.ts

@@ -0,0 +1,53 @@
+import { FileType, ProjectPluginStructure } from '@teleporthq/teleport-types'
+import { getDatabaseDriverDependencies } from '../auth-generator'
+import { resolveInvoiceDataSource } from '../invoice/resolve-data-source'
+import {
+  generateSecurityRuntimeCode,
+  SECURITY_RUNTIME_FILE_KEY,
+  SECURITY_RUNTIME_FILE_NAME,
+  SECURITY_RUNTIME_PATH,
+} from './security-runtime-code'
+
+export {
+  generateSecurityRuntimeCode,
+  RATE_LIMITS_TABLE,
+  SECURITY_RUNTIME_FILE_KEY,
+  SECURITY_RUNTIME_FILE_NAME,
+  SECURITY_RUNTIME_PATH,
+  securityRuntimeSpecifier,
+  TOO_MANY_REQUESTS_MESSAGE,
+} from './security-runtime-code'
+
+/**
+ * Writes `utils/security-runtime.js` into the project, once. The workflow
+ * project plugin calls it for every app with server routes; an emitter whose
+ * route can exist in an app without them calls it where it emits the route
+ * that `require`s the module.
+ *
+ * The module is the same for every app (it finds the database at run time);
+ * the `pg` driver is added only when the project's database is Postgres,
+ * through the same driver map auth and invoices use.
+ */
+export const ensureSecurityRuntimeModule = (structure: ProjectPluginStructure): void => {
+  const { files, dependencies } = structure
+  if (files.has(SECURITY_RUNTIME_FILE_KEY)) {
+    return
+  }
+
+  files.set(SECURITY_RUNTIME_FILE_KEY, {
+    path: SECURITY_RUNTIME_PATH,
+    files: [
+      {
+        name: SECURITY_RUNTIME_FILE_NAME,
+        fileType: FileType.JS,
+        content: generateSecurityRuntimeCode(),
+      },
+    ],
+  })
+
+  const { dataSourceType } = resolveInvoiceDataSource(structure)
+  const driverDeps = getDatabaseDriverDependencies(dataSourceType)
+  if ('pg' in driverDeps && !dependencies.pg) {
+    dependencies.pg = driverDeps.pg
+  }
+}