@produtype/core 0.38.0 → 0.40.0

package/dist/analyzer/detectSecurity.js

@@ -5,6 +5,43 @@ const detectContext_1 = require("./detectContext");
 const readTextFileSafe_1 = require("../utils/readTextFileSafe");
 const textSearch_1 = require("../utils/textSearch");
 const absenceEvidence_1 = require("./absenceEvidence");
+/** Lines that decide which origins may call this server. */
+const ORIGIN_HANDLING = [/Access-Control-Allow-Origin/i, /ALLOWED_ORIGINS/, /allowedOrigins/i];
+/**
+ * The same line, allowing everyone.
+ *
+ * Both spellings, because `'Access-Control-Allow-Origin', '*'` and
+ * `ALLOWED_ORIGINS = ["*"]` are the same decision written in two frameworks, and only
+ * the first was being caught.
+ */
+const WILDCARD_ORIGIN = /(Access-Control-Allow-Origin["'\s:,]+\*)|(["']\*["'])/i;
+/**
+ * Flask's answer, which was invisible.
+ *
+ * `\bcors\s*\(` is case-sensitive, so `CORS(app)` matched nothing, and the extension's
+ * default is to allow every origin — the one configuration this check exists to find.
+ * A Flask application that opens itself to the whole web in one line was reported as
+ * having no cross-origin configuration at all.
+ */
+const IMPORTS_FLASK_CORS = /from\s+flask_cors\s+import|import\s+flask_cors/;
+const FLASK_CORS_CALL = /\bCORS\s*\(/;
+/**
+ * The line with its quoted text removed.
+ *
+ * `cors(` inside a string literal is never the middleware being applied — it is prose
+ * about it. This tool's own remediation catalogue, "Replace cors() defaults with an
+ * explicit allowlist.", was being cited as a CORS configuration, and no rule about the
+ * shape of the line could help: a standalone string in a list is deliberately not
+ * skipped, because `'django.contrib.auth',` in INSTALLED_APPS is the behaviour itself.
+ *
+ * Narrow on purpose. It is applied to this one question, where a call is what is being
+ * looked for, and not to the file search, where a string is often the answer.
+ */
+function withoutStringLiterals(line) {
+    return line.replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, '""');
+}
+/** The Express middleware, brought into the file that configures it. */
+const IMPORTS_CORS = /(require\(['"]cors['"]\))|(from\s+['"]cors['"])|(import\s+['"]cors['"])/;
 function detectCorsConfig(text, file) {
     const loose = [];
     const strict = [];
@@ -12,7 +49,9 @@ function detectCorsConfig(text, file) {
     const corsVarRegex = /\bcors\(\s*([A-Za-z_$][\w$]*)\s*\)/;
     for (let i = 0; i < lines.length; i++) {
         const line = lines[i];
-        if (!/\bcors\s*\(/.test(line))
+        if (!(0, textSearch_1.isCitableLine)(line))
+            continue;
+        if (!/\bcors\s*\(/.test(withoutStringLiterals(line)))
             continue;
         const snippet = line.trim().slice(0, 200);
         if (/\bcors\(\s*\)/.test(line)) {
@@ -41,6 +80,35 @@ function detectCorsConfig(text, file) {
     }
     return { loose, strict };
 }
+/**
+ * A settings file that is Django's, rather than one that shares its name.
+ *
+ * `diet_hub/api/settings.py` is a FastAPI router that lets an administrator change
+ * application options from the browser. It was read as Django's configuration, and on
+ * the strength of the filename alone the project was credited with having security
+ * middleware it does not have — the one direction of error that matters here, because
+ * it hides a missing control rather than inventing a present one.
+ *
+ * Two conditions, because either alone is wrong. A repository can depend on Django and
+ * still own a dozen files called settings.py; a file can declare INSTALLED_APPS in a
+ * tutorial that the product does not run. And the first file named settings.py is not
+ * the right one: a project with `settings/base.py` and `settings/production.py` has
+ * several, so every candidate is examined and the first that is Django's is used.
+ */
+async function findDjangoSettings(ctx) {
+    if (!ctx.pythonDeps.includes('django'))
+        return null;
+    const candidates = ctx.files.all.filter((f) => /(^|\/)settings(_[a-z]+)?\.py$/i.test(f) || /(^|\/)settings\/[a-z_]+\.py$/i.test(f));
+    for (const file of candidates) {
+        const text = await (0, readTextFileSafe_1.readTextFileSafe)(ctx.root, file);
+        if (!text)
+            continue;
+        if (/^\s*INSTALLED_APPS\s*=/m.test(text) || /^\s*MIDDLEWARE\s*=/m.test(text) || /DJANGO_SETTINGS_MODULE/.test(text)) {
+            return { file, text };
+        }
+    }
+    return null;
+}
 async function detectSecurity(ctx) {
     const evidence = [];
     const source = ctx.files.source;
@@ -82,14 +150,35 @@ async function detectSecurity(ctx) {
             continue;
         // Framework-native CORS: an explicit allowlist checked against the Origin header,
         // rather than the Express cors() middleware.
-        if (/Access-Control-Allow-Origin/i.test(text) || /ALLOWED_ORIGINS/.test(text) || /allowedOrigins/i.test(text)) {
-            const wildcard = /Access-Control-Allow-Origin["'\s:,]+\*/i.test(text);
-            const hit = { snippet: 'explicit origin handling', file, line: 1 };
-            if (wildcard)
-                corsLoose.push(hit);
+        //
+        // Line by line, because the file is the wrong unit for this question twice over.
+        // It decided loose-or-strict from whether a wildcard appeared anywhere in the
+        // file, so one permissive route made an allowlist read as permissive and one
+        // allowlist made a wildcard read as restricted; and it cited line 1, which in
+        // every repository that triggered it was an import.
+        for (const m of (0, textSearch_1.matchLines)(text, ORIGIN_HANDLING, file)) {
+            if (WILDCARD_ORIGIN.test(m.snippet))
+                corsLoose.push(m);
             else
-                corsStrict.push(hit);
+                corsStrict.push(m);
+        }
+        // Only where the middleware is actually imported. A sentence in this tool's own
+        // remediation catalogue — "Replace cors() defaults with an explicit allowlist." —
+        // was being read as a CORS configuration, and a table of strings is not a comment,
+        // so no rule about the shape of the line could tell them apart. Whether the file
+        // imports the package can.
+        if (IMPORTS_FLASK_CORS.test(text)) {
+            const lines = text.split(/\r?\n/);
+            for (const m of (0, textSearch_1.matchLines)(text, [FLASK_CORS_CALL], file).filter((m) => FLASK_CORS_CALL.test(withoutStringLiterals(m.snippet)))) {
+                const window = lines.slice(m.line - 1, m.line + 9).join('\n');
+                if (/\borigins\s*=/.test(window) || /\bresources\s*=/.test(window))
+                    corsStrict.push(m);
+                else
+                    corsLoose.push(m);
+            }
         }
+        if (!IMPORTS_CORS.test(text))
+            continue;
         if (!/\bcors\s*\(/.test(text))
             continue;
         const detected = detectCorsConfig(text, file);
@@ -125,27 +214,36 @@ async function detectSecurity(ctx) {
         evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'body-size' });
     for (const m of contentTypeCheck)
         evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'content-type' });
-    const djangoSettings = ctx.files.all.find((f) => f.endsWith('settings.py'));
+    /**
+     * Django was credited for headers twice, and neither credit was earned.
+     *
+     * The first was the mere existence of a file named settings.py, which a FastAPI
+     * router satisfied. The second looked truer — `SecurityMiddleware` really does set
+     * those headers — but `django-admin startproject` writes that line into every new
+     * project, so it distinguishes nothing and would have credited the bare skeleton
+     * this repository keeps as a fixture precisely because it is unprotected. What is
+     * left is the `SECURE_*` settings above: lines somebody chose to write.
+     */
+    const djangoSettings = await findDjangoSettings(ctx);
     let djangoDebugTrue = false;
     let djangoSecureCookies = true;
     if (djangoSettings) {
-        const text = (await (0, readTextFileSafe_1.readTextFileSafe)(ctx.root, djangoSettings)) ?? '';
-        if (/DEBUG\s*=\s*True/.test(text)) {
+        const { file, text } = djangoSettings;
+        const debugOn = (0, textSearch_1.matchLines)(text, [/^\s*DEBUG\s*=\s*True\b/], file);
+        if (debugOn.length > 0) {
             djangoDebugTrue = true;
-            evidence.push({ type: 'snippet', value: 'DEBUG = True', file: djangoSettings, claim: 'django-debug' });
+            for (const m of debugOn)
+                evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'django-debug' });
         }
-        const insecureSession = /SESSION_COOKIE_SECURE\s*=\s*False/.test(text);
-        const insecureCsrf = /CSRF_COOKIE_SECURE\s*=\s*False/.test(text);
-        djangoSecureCookies = !(insecureSession || insecureCsrf);
-        if (insecureSession)
-            evidence.push({ type: 'snippet', value: 'SESSION_COOKIE_SECURE = False', file: djangoSettings, claim: 'django-cookies' });
-        if (insecureCsrf)
-            evidence.push({ type: 'snippet', value: 'CSRF_COOKIE_SECURE = False', file: djangoSettings, claim: 'django-cookies' });
+        const insecure = (0, textSearch_1.matchLines)(text, [/^\s*SESSION_COOKIE_SECURE\s*=\s*False\b/, /^\s*CSRF_COOKIE_SECURE\s*=\s*False\b/], file);
+        djangoSecureCookies = insecure.length === 0;
+        for (const m of insecure)
+            evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'django-cookies' });
     }
     return {
         key: 'security.core',
-        present: helmet || rateLimit || Boolean(djangoSettings),
-        complete: helmet && (rateLimit || Boolean(djangoSettings)),
+        present: helmet || rateLimit,
+        complete: helmet && rateLimit,
         evidence,
         details: {
             helmet,

package/dist/analyzer/detectUploads.js

@@ -25,6 +25,84 @@ function findExpressUploadsRoutes(text, file) {
     }
     return out;
 }
+/**
+ * The lines of a Python file that only run in development.
+ *
+ * `urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)` is the
+ * line Django's own documentation gives for serving uploaded files, and the
+ * documentation wraps it in `if settings.DEBUG:` — which is the whole point of the
+ * snippet. This detector already said so in a comment and then counted the line anyway,
+ * so a travel application was told its uploads were exposed to the public by a line
+ * that does not run in production.
+ *
+ * Block extent by indentation, which is how Python says it.
+ */
+function debugOnlyLines(text) {
+    const guarded = new Set();
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const opener = lines[i].match(/^(\s*)if\s+(?:settings\.)?DEBUG\s*:/);
+        if (!opener)
+            continue;
+        const indent = opener[1].length;
+        for (let j = i + 1; j < lines.length; j++) {
+            const line = lines[j];
+            if (line.trim() === '') {
+                guarded.add(j + 1);
+                continue;
+            }
+            const lineIndent = line.length - line.trimStart().length;
+            if (lineIndent <= indent)
+                break;
+            guarded.add(j + 1);
+        }
+    }
+    return guarded;
+}
+/**
+ * A URL pattern that serves uploaded files, as distinct from one that serves the
+ * stylesheet.
+ *
+ * `document_root=` alone matched `static(settings.STATIC_URL, document_root=STATIC_ROOT)`
+ * — CSS and JavaScript, which every Django project serves and nobody uploads. Two of
+ * the projects reported as exposing their uploads were exposing their stylesheets.
+ */
+const DJANGO_MEDIA_ROUTES = [
+    /static\s*\(\s*settings\.MEDIA_URL/i,
+    /document_root\s*=\s*settings\.MEDIA_ROOT/i,
+    /re_path\s*\(\s*r?['"][^'"]*media/i,
+    /url\s*\(\s*r?['"][^'"]*media/i,
+];
+/**
+ * Checking the type of a file somebody uploaded.
+ *
+ * This was `/mime/i` and `/content-type/i`, which is every JSON response header in
+ * every project: `res.writeHead(404, { 'Content-Type': 'application/json' })` was
+ * cited as upload validation, and so was `X-Content-Type-Options: nosniff`. The flag
+ * came out true for all six projects in the corpus that handle uploads — a measurement
+ * that never varies is not one — and it decides whether upload protection reads
+ * `present` or `partial`, so two projects were credited for a header on a 404.
+ */
+const DJANGO_VIEW_PROTECTION = [
+    /@login_required/,
+    /LoginRequiredMixin/,
+    /PermissionRequiredMixin/,
+    /@user_passes_test/,
+    /@permission_required/,
+];
+/** A file that does something with an uploaded file, rather than one that logs people in. */
+const HANDLES_FILES = /request\.FILES|FileField|ImageField|\bupload/i;
+const UPLOAD_VALIDATION = [
+    /\bfileFilter\b/,
+    /file\.mimetype/i,
+    /files?\[[^\]]*\]\.mimetype/i,
+    /allowed(?:File|Mime|Content)?_?(?:Types|Extensions)/i,
+    /FileExtensionValidator/,
+    /validate_image_file_extension/,
+    /\bfileTypeFrom(?:Buffer|File|Stream)\b/,
+    /request\.FILES\[[^\]]*\]\.content_type/,
+    /\.content_type\s*(?:not\s+)?in\b/,
+];
 async function detectUploads(ctx) {
     const evidence = [];
     const uploadDeps = (0, detectContext_1.hasAnyDep)(ctx, ['multer', 'formidable', '@aws-sdk/client-s3', 'aws-sdk']);
@@ -65,28 +143,35 @@ async function detectUploads(ctx) {
      * A real report on a real school platform called its uploads publicly exposed on the
      * strength of those two lines, with no pattern serving media anywhere in the project.
      */
-    const djangoPublicSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [
-        /static\s*\(\s*settings\.MEDIA_URL/i,
-        /document_root\s*=/i,
-        /re_path\s*\(\s*r?['"][^'"]*media/i,
-        /url\s*\(\s*r?['"][^'"]*media/i,
-    ], 15);
-    /**
-     * Django's own way of protecting a view: a decorator or a mixin, not middleware on a
-     * route. The Express-shaped route scan cannot see either, so a project whose upload
-     * view is `@login_required` read as having no protection at all.
-     */
-    const djangoProtectionSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [
-        /@login_required/,
-        /LoginRequiredMixin/,
-        /PermissionRequiredMixin/,
-        /@user_passes_test/,
-        /@permission_required/,
-    ], 15);
+    const djangoPublicSignals = [];
+    const djangoProtectionSignals = [];
+    const validationSignals = [];
+    for (const file of source) {
+        const text = await (0, readTextFileSafe_1.readTextFileSafe)(ctx.root, file);
+        if (!text)
+            continue;
+        validationSignals.push(...(0, textSearch_1.matchLines)(text, UPLOAD_VALIDATION, file));
+        if (!file.endsWith('.py'))
+            continue;
+        const developmentOnly = debugOnlyLines(text);
+        djangoPublicSignals.push(...(0, textSearch_1.matchLines)(text, DJANGO_MEDIA_ROUTES, file).filter((m) => !developmentOnly.has(m.line)));
+        /**
+         * Django's own way of protecting a view: a decorator or a mixin, not middleware on
+         * a route. The Express-shaped route scan cannot see either, so a project whose
+         * upload view is `@login_required` read as having no protection at all.
+         *
+         * Only in a file that handles files. This detector argues, three comments above,
+         * that authentication is not evidence about uploads — and then counted every
+         * `@login_required` in the project as upload protection: a photography business
+         * was credited for the decorators on its accounting views.
+         */
+        if (HANDLES_FILES.test(text)) {
+            djangoProtectionSignals.push(...(0, textSearch_1.matchLines)(text, DJANGO_VIEW_PROTECTION, file));
+        }
+    }
     for (const m of djangoProtectionSignals) {
         evidence.push({ type: 'snippet', value: m.snippet, file: m.file, line: m.line, claim: 'protection' });
     }
-    const validationSignals = await (0, textSearch_1.searchInFiles)(ctx.root, source, [/file-type/i, /mime/i, /content-type/i], 15);
     const protectedRoutes = routeSignals.filter((r) => r.protected);
     const unprotectedRoutes = routeSignals.filter((r) => !r.protected);
     for (const r of routeSignals) {

package/dist/rules/rules.js

@@ -390,9 +390,14 @@ exports.rules = [
             const loose = Boolean(sec?.details?.corsLoose);
             const strict = Boolean(sec?.details?.corsStrict);
             // Nothing answers a cross-origin request where nothing answers a request.
+            //
+            // The permissive line decides, not the restricted one. While the file was the
+            // unit of judgement this hardly mattered; now that each line is read on its own,
+            // an allowlist on one route would otherwise absolve a wildcard on another — and
+            // a wildcard is open to the whole web no matter what the route beside it does.
             const status = !hasUserFacingSurface(analysis)
                 ? 'unknown'
-                : strict ? 'passed' : loose ? 'partial' : 'unknown';
+                : loose ? 'partial' : strict ? 'passed' : 'unknown';
             return mkFinding({
                 id: 'security.cors-origin',
                 title: 'CORS origin restrictions',
@@ -402,7 +407,9 @@ exports.rules = [
                 description: status === 'passed'
                     ? 'CORS appears configured with explicit origins.'
                     : status === 'partial'
-                        ? 'CORS middleware detected without explicit origin restrictions.'
+                        ? strict
+                            ? 'Some routes restrict origins and at least one allows any origin.'
+                            : 'CORS middleware detected without explicit origin restrictions.'
                         : 'CORS configuration not detected.',
                 recommendation: 'Configure allowlist origins and avoid permissive defaults in production.',
                 evidence: evidenceForClaim(sec?.evidence, 'cors'),

package/dist/utils/textSearch.d.ts

@@ -3,6 +3,27 @@ export interface TextMatch {
     line: number;
     snippet: string;
 }
+/**
+ * Whether this line may be quoted back to the reader as something the code does.
+ *
+ * The rules above were reachable only through the file search, so any detector that
+ * walked lines itself got none of them. The CORS middleware walk was one: it cited
+ * `// app.use(cors({` from a route somebody had commented out, and — pointed at this
+ * repository — cited this tool's own prose about the Express middleware and the
+ * sentence in its remediation catalogue that advises replacing `cors()` defaults.
+ * Three citations, none of them code that runs.
+ */
+export declare function isCitableLine(line: string): boolean;
+/**
+ * The lines of one already-read file that match, with the same hygiene the file
+ * search applies: no comments, no pattern tables, no minified lines.
+ *
+ * Exported because a detector that has the text in hand should not have to choose
+ * between re-reading the file and skipping those rules. The CORS branch made that
+ * choice the wrong way: it tested the whole file for a header name and cited line 1
+ * with the words "explicit origin handling" — a conclusion where a line was promised.
+ */
+export declare function matchLines(text: string, needles: Array<string | RegExp>, file?: string): TextMatch[];
 /**
  * Search a list of relative file paths for any of the given needles
  * (string or RegExp). Returns at most `limit` matches.

package/dist/utils/textSearch.js

@@ -1,5 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.isCitableLine = isCitableLine;
+exports.matchLines = matchLines;
 exports.searchInFiles = searchInFiles;
 exports.anyIncludes = anyIncludes;
 const readTextFileSafe_1 = require("./readTextFileSafe");
@@ -49,6 +51,19 @@ function declaresRatherThanDoes(line) {
     const trimmed = line.trim();
     return COMMENT_LINE.test(trimmed) || PATTERN_DECLARATION.test(trimmed);
 }
+/**
+ * Whether this line may be quoted back to the reader as something the code does.
+ *
+ * The rules above were reachable only through the file search, so any detector that
+ * walked lines itself got none of them. The CORS middleware walk was one: it cited
+ * `// app.use(cors({` from a route somebody had commented out, and — pointed at this
+ * repository — cited this tool's own prose about the Express middleware and the
+ * sentence in its remediation catalogue that advises replacing `cors()` defaults.
+ * Three citations, none of them code that runs.
+ */
+function isCitableLine(line) {
+    return line.length <= MAX_CITABLE_LINE && !declaresRatherThanDoes(line);
+}
 /**
  * The longest line this analyzer will cite.
  *
@@ -69,6 +84,34 @@ function declaresRatherThanDoes(line) {
  * on is not worth the one it displaces.
  */
 const MAX_CITABLE_LINE = 500;
+/**
+ * The lines of one already-read file that match, with the same hygiene the file
+ * search applies: no comments, no pattern tables, no minified lines.
+ *
+ * Exported because a detector that has the text in hand should not have to choose
+ * between re-reading the file and skipping those rules. The CORS branch made that
+ * choice the wrong way: it tested the whole file for a header name and cited line 1
+ * with the words "explicit origin handling" — a conclusion where a line was promised.
+ */
+function matchLines(text, needles, file = '') {
+    const matches = [];
+    const lines = text.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.length > MAX_CITABLE_LINE)
+            continue;
+        if (declaresRatherThanDoes(line))
+            continue;
+        for (const n of needles) {
+            const hit = typeof n === 'string' ? line.includes(n) : n.test(line);
+            if (hit) {
+                matches.push({ file, line: i + 1, snippet: line.trim().slice(0, 200) });
+                break;
+            }
+        }
+    }
+    return matches;
+}
 /**
  * Search a list of relative file paths for any of the given needles
  * (string or RegExp). Returns at most `limit` matches.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@produtype/core",
-  "version": "0.38.0",
+  "version": "0.40.0",
   "description": "Deterministic CLI and library that analyzes a web application repository and reports how far it is from production-ready for the kind of product it is meant to be.",
   "license": "MIT",
   "bin": {