@produtype/core 0.39.0 → 0.41.0

package/dist/analyzer/detectEnv.js

@@ -45,6 +45,26 @@ function namesItself(snippet) {
     const flatten = (value) => value.toLowerCase().replace(/[^a-z0-9]/g, '');
     return flatten(assignment[1]) === flatten(assignment[2]);
 }
+/**
+ * The value is the name of something, spelled the way every language spells a name.
+ *
+ * `REDIS_SECRET_KEY = "SECRET_TOKEN"` in Discourse is the Redis key under which the
+ * secret is stored — the comment directly above it says so — and it was reported as a
+ * hardcoded secret at `critical`, the severity that bars a report from the top band.
+ * `namesItself` could not catch it: the constant and its value are two different
+ * names, so the letters do not match.
+ *
+ * What settles it is the shape. `SECRET_TOKEN` is SCREAMING_SNAKE_CASE, which is how
+ * constants, environment variables and Redis keys are written and not how secrets are
+ * written — a secret is entropy, and this has none. Kept to that one casing on
+ * purpose: `dev-secret` and `changeme` are lowercase and stay caught.
+ */
+function valueIsAnIdentifier(snippet) {
+    const assignment = /([A-Za-z_][A-Za-z0-9_]*)\s*[:=]\s*["'`]([^"'`]+)["'`]/.exec(snippet);
+    if (!assignment)
+        return false;
+    return /^[A-Z][A-Z0-9]*(_[A-Z0-9]+)+$/.test(assignment[2]);
+}
 async function detectEnv(ctx) {
     const evidence = [];
     const weakSecretEvidence = [];
@@ -98,6 +118,7 @@ async function detectEnv(ctx) {
     const weakHits = fallbackHits.filter((m) => WEAK_SECRET_VALUE_RE.test(m.snippet)
         && SECRET_ASSIGNMENT_CONTEXT_RE.test(m.snippet)
         && !namesItself(m.snippet)
+        && !valueIsAnIdentifier(m.snippet)
         && !isTableEntry(m));
     for (const m of weakHits) {
         const hitEvidence = { type: 'snippet', value: m.snippet, file: m.file, line: m.line };

package/dist/analyzer/detectSecurity.js

@@ -5,6 +5,7 @@ const detectContext_1 = require("./detectContext");
 const readTextFileSafe_1 = require("../utils/readTextFileSafe");
 const textSearch_1 = require("../utils/textSearch");
 const absenceEvidence_1 = require("./absenceEvidence");
+const developmentOnly_1 = require("./developmentOnly");
 /** Lines that decide which origins may call this server. */
 const ORIGIN_HANDLING = [/Access-Control-Allow-Origin/i, /ALLOWED_ORIGINS/, /allowedOrigins/i];
 /**
@@ -145,6 +146,10 @@ async function detectSecurity(ctx) {
     const corsLoose = [];
     const corsStrict = [];
     for (const file of source) {
+        // Rails picks one environment file by RAILS_ENV, so a header set in
+        // development.rb is not a header this product sends.
+        if ((0, developmentOnly_1.isDevelopmentOnlyFile)(file))
+            continue;
         const text = await (0, readTextFileSafe_1.readTextFileSafe)(ctx.root, file);
         if (!text)
             continue;

package/dist/analyzer/detectUploads.js

@@ -25,6 +25,84 @@ function findExpressUploadsRoutes(text, file) {
     }
     return out;
 }
+/**
+ * The lines of a Python file that only run in development.
+ *
+ * `urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)` is the
+ * line Django's own documentation gives for serving uploaded files, and the
+ * documentation wraps it in `if settings.DEBUG:` — which is the whole point of the
+ * snippet. This detector already said so in a comment and then counted the line anyway,
+ * so a travel application was told its uploads were exposed to the public by a line
+ * that does not run in production.
+ *
+ * Block extent by indentation, which is how Python says it.
+ */
+function debugOnlyLines(text) {
+    const guarded = new Set();
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const opener = lines[i].match(/^(\s*)if\s+(?:settings\.)?DEBUG\s*:/);
+        if (!opener)
+            continue;
+        const indent = opener[1].length;
+        for (let j = i + 1; j < lines.length; j++) {
+            const line = lines[j];
+            if (line.trim() === '') {
+                guarded.add(j + 1);
+                continue;
+            }
+            const lineIndent = line.length - line.trimStart().length;
+            if (lineIndent <= indent)
+                break;
+            guarded.add(j + 1);
+        }
+    }
+    return guarded;
+}
+/**
+ * A URL pattern that serves uploaded files, as distinct from one that serves the
+ * stylesheet.
+ *
+ * `document_root=` alone matched `static(settings.STATIC_URL, document_root=STATIC_ROOT)`
+ * — CSS and JavaScript, which every Django project serves and nobody uploads. Two of
+ * the projects reported as exposing their uploads were exposing their stylesheets.
+ */
+const DJANGO_MEDIA_ROUTES = [
+    /static\s*\(\s*settings\.MEDIA_URL/i,
+    /document_root\s*=\s*settings\.MEDIA_ROOT/i,
+    /re_path\s*\(\s*r?['"][^'"]*media/i,
+    /url\s*\(\s*r?['"][^'"]*media/i,
+];
+/**
+ * Checking the type of a file somebody uploaded.
+ *
+ * This was `/mime/i` and `/content-type/i`, which is every JSON response header in
+ * every project: `res.writeHead(404, { 'Content-Type': 'application/json' })` was
+ * cited as upload validation, and so was `X-Content-Type-Options: nosniff`. The flag
+ * came out true for all six projects in the corpus that handle uploads — a measurement
+ * that never varies is not one — and it decides whether upload protection reads
+ * `present` or `partial`, so two projects were credited for a header on a 404.
+ */
+const DJANGO_VIEW_PROTECTION = [
+    /@login_required/,
+    /LoginRequiredMixin/,
+    /PermissionRequiredMixin/,
+    /@user_passes_test/,
+    /@permission_required/,
+];
+/** A file that does something with an uploaded file, rather than one that logs people in. */
+const HANDLES_FILES = /request\.FILES|FileField|ImageField|\bupload/i;
+const UPLOAD_VALIDATION = [
+    /\bfileFilter\b/,
+    /file\.mimetype/i,
+    /files?\[[^\]]*\]\.mimetype/i,
+    /allowed(?:File|Mime|Content)?_?(?:Types|Extensions)/i,
+    /FileExtensionValidator/,
+    /validate_image_file_extension/,
+    /\bfileTypeFrom(?:Buffer|File|Stream)\b/,
+    /request\.FILES\[[^\]]*\]\.content_type/,
+    /\.content_type\s*(?:not\s+)?in\b/,
+];
 async function detectUploads(ctx) {
     const evidence = [];
     const uploadDeps = (0, detectContext_1.hasAnyDep)(ctx, ['multer', 'formidable', '@aws-sdk/client-s3', 'aws-sdk']);
@@ -65,28 +143,35 @@ async function detectUploads(ctx) {
      * A real report on a real school platform called its uploads publicly exposed on the
      * strength of those two lines, with no pattern serving media anywhere in the project.
      */
-    const djangoPublicSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [
-        /static\s*\(\s*settings\.MEDIA_URL/i,
-        /document_root\s*=/i,
-        /re_path\s*\(\s*r?['"][^'"]*media/i,
-        /url\s*\(\s*r?['"][^'"]*media/i,
-    ], 15);
-    /**
-     * Django's own way of protecting a view: a decorator or a mixin, not middleware on a
-     * route. The Express-shaped route scan cannot see either, so a project whose upload
-     * view is `@login_required` read as having no protection at all.
-     */
-    const djangoProtectionSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [
-        /@login_required/,
-        /LoginRequiredMixin/,
-        /PermissionRequiredMixin/,
-        /@user_passes_test/,
-        /@permission_required/,
-    ], 15);
+    const djangoPublicSignals = [];
+    const djangoProtectionSignals = [];
+    const validationSignals = [];
+    for (const file of source) {
+        const text = await (0, readTextFileSafe_1.readTextFileSafe)(ctx.root, file);
+        if (!text)
+            continue;
+        validationSignals.push(...(0, textSearch_1.matchLines)(text, UPLOAD_VALIDATION, file));
+        if (!file.endsWith('.py'))
+            continue;
+        const developmentOnly = debugOnlyLines(text);
+        djangoPublicSignals.push(...(0, textSearch_1.matchLines)(text, DJANGO_MEDIA_ROUTES, file).filter((m) => !developmentOnly.has(m.line)));
+        /**
+         * Django's own way of protecting a view: a decorator or a mixin, not middleware on
+         * a route. The Express-shaped route scan cannot see either, so a project whose
+         * upload view is `@login_required` read as having no protection at all.
+         *
+         * Only in a file that handles files. This detector argues, three comments above,
+         * that authentication is not evidence about uploads — and then counted every
+         * `@login_required` in the project as upload protection: a photography business
+         * was credited for the decorators on its accounting views.
+         */
+        if (HANDLES_FILES.test(text)) {
+            djangoProtectionSignals.push(...(0, textSearch_1.matchLines)(text, DJANGO_VIEW_PROTECTION, file));
+        }
+    }
     for (const m of djangoProtectionSignals) {
         evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'protection' });
     }
-    const validationSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [/file-type/i, /mime/i, /content-type/i], 15);
     const protectedRoutes = routeSignals.filter((r) => r.protected);
     const unprotectedRoutes = routeSignals.filter((r) => !r.protected);
     for (const r of routeSignals) {

package/dist/analyzer/developmentOnly.d.ts

@@ -0,0 +1 @@
+export declare function isDevelopmentOnlyFile(file: string): boolean;

package/dist/analyzer/developmentOnly.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isDevelopmentOnlyFile = isDevelopmentOnlyFile;
+/**
+ * A file the product does not run in production.
+ *
+ * Discourse was reported as allowing every cross-origin request, and one of the two
+ * lines behind it was `config/environments/development.rb:34`. Rails loads exactly one
+ * of those files, chosen by `RAILS_ENV`, so a header set in `development.rb` is as
+ * absent from production as a line inside `if settings.DEBUG:` — the same mistake this
+ * analyzer had already made in Python and fixed there.
+ *
+ * Deliberately only the frameworks that name the environment in the path, because
+ * that is the only case where the file's own name settles the question. A file called
+ * `dev-server.js` might be anything; `config/environments/test.rb` cannot be.
+ */
+const DEVELOPMENT_ONLY_PATHS = [
+    /(^|\/)config\/environments\/(?!production)[a-z_]+\.rb$/i,
+    /(^|\/)config\/environments\/(?!production)[a-z_]+\.exs$/i,
+];
+function isDevelopmentOnlyFile(file) {
+    return DEVELOPMENT_ONLY_PATHS.some((pattern) => pattern.test(file));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@produtype/core",
-  "version": "0.39.0",
+  "version": "0.41.0",
   "description": "Deterministic CLI and library that analyzes a web application repository and reports how far it is from production-ready for the kind of product it is meant to be.",
   "license": "MIT",
   "bin": {