@haystackeditor/cli 0.24.1 → 0.25.1

package/dist/capture/csp.js

@@ -0,0 +1,332 @@
+/**
+ * Content-Security-Policy (CAPTURE-V1 rule 8b(iii)): init finds the app's policy where the repository keeps it (headers
+ * config, middleware, meta tags, framework CSP options, server configs) and reports the exact additions the tag needs;
+ * it never edits or loosens a policy, and never proposes a wildcard or 'unsafe-inline'.
+ *
+ * The tag needs: connect-src 'self' (the manifest, fetched from the app's own origin) and https://c.haystack.sh (the
+ * ingest); script-src 'self' (the self-hosted files). A connect-src that is absent falls back to default-src, so adding
+ * one copies default-src's sources (otherwise everything else default-src allowed for fetches would break).
+ * 'strict-dynamic' ignores 'self': Next.js's Script loader is itself a trusted script and may insert the tag; a plain
+ * <script> then needs the page's nonce.
+ */
+import fg from 'fast-glob';
+import { readFileSync, statSync } from 'node:fs';
+import { extname, join, relative } from 'node:path';
+import { SAXParser } from 'parse5-sax-parser';
+import { CAPTURE_ORIGIN } from '../commands/capture-contract.js';
+import { keyName, parseFile, parseJsonLike, unwrapTs, walk } from './js-ast.js';
+import { toPosix } from './project.js';
+const SCAN = ['**/*.{js,mjs,cjs,ts,mts,cts,jsx,tsx,json,toml,html,erb,rb,py,conf,yaml,yml}', '**/_headers'];
+const IGNORE = ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/.next/**', '**/out/**', '**/vendor/**',
+    '**/.svelte-kit/**', '**/.nuxt/**', '**/.output/**', '**/coverage/**', '**/tmp/**', '**/.venv/**', '**/venv/**',
+    '**/site-packages/**', '**/_haystack/**', '**/*.test.*', '**/*.spec.*', '**/__tests__/**', '**/package-lock.json',
+    '**/*.min.js'];
+const MARKER = /content[-_]?security[-_]?policy|\bCSP_[A-Z_]+_SRC\b|\bcsp\s*:|\bhelmet\b/i;
+const DIRECTIVE = /\b(default|script|script-src-elem|connect|style|img|font|frame|object|base|form|worker|media|manifest)-src\b|\bscript-src-elem\b/;
+const MAX_FILE_BYTES = 512 * 1024;
+const DYNAMIC = '<dynamic>';
+function parsePolicy(policy) {
+    const directives = new Map();
+    for (const part of policy.split(';')) {
+        const [name, ...sources] = part.trim().split(/\s+/);
+        if (!name || directives.has(name.toLowerCase()))
+            continue;
+        directives.set(name.toLowerCase(), sources);
+    }
+    return directives;
+}
+function allowsCaptureOrigin(sources) {
+    return sources.some(source => ['*', 'https:', CAPTURE_ORIGIN, 'c.haystack.sh', 'https://*.haystack.sh', '*.haystack.sh'].includes(source.toLowerCase()));
+}
+/** Does the list allow the page's own origin? Production origins are https (rule 8b), so `https:` and `*` cover it, as
+ * does naming every production origin. */
+function allowsSelf(sources, origins) {
+    return sources.some(source => source === '\'self\'' || source === '*' || source === 'https:')
+        || (origins.length > 0 && origins.every(origin => sources.includes(origin)));
+}
+/** The additions one readable policy needs for the tag on these production origins. */
+export function analyzePolicy(policy, nextLoader, origins) {
+    const directives = parsePolicy(policy);
+    const additions = [];
+    const notes = [];
+    const defaults = directives.get('default-src');
+    const connect = directives.get('connect-src');
+    const connectSources = connect ?? defaults;
+    if (connectSources) {
+        const missing = [...(allowsSelf(connectSources, origins) ? [] : ['\'self\'']), ...(allowsCaptureOrigin(connectSources) ? [] : [CAPTURE_ORIGIN])];
+        if (missing.length > 0) {
+            const kept = connectSources.filter(source => source !== '\'none\'');
+            additions.push(connect
+                ? `connect-src ${[...kept, ...missing].join(' ')}   (add ${missing.join(' ')})`
+                : `connect-src ${[...kept, ...missing].join(' ')}   (a new directive: default-src's sources plus ${missing.join(' ')})`);
+        }
+    }
+    const scriptName = directives.has('script-src-elem') ? 'script-src-elem' : directives.has('script-src') ? 'script-src' : directives.has('default-src') ? 'default-src' : null;
+    const scriptSources = scriptName ? directives.get(scriptName) : null;
+    if (scriptSources && scriptSources.includes('\'strict-dynamic\'')) {
+        notes.push(nextLoader
+            ? `${scriptName} uses 'strict-dynamic': Next.js's Script loader inserts the tag, which 'strict-dynamic' allows while Next's own scripts carry the nonce.`
+            : `${scriptName} uses 'strict-dynamic', which ignores 'self': give each Haystack <script> the page's nonce attribute.`);
+        if (!nextLoader)
+            additions.push(`nonce on the Haystack <script> tags   (${scriptName} uses 'strict-dynamic')`);
+    }
+    else if (scriptSources && !allowsSelf(scriptSources, origins)) {
+        const kept = scriptSources.filter(source => source !== '\'none\'');
+        additions.push(scriptName === 'default-src'
+            ? `script-src ${[...kept, '\'self\''].join(' ')}   (a new directive: default-src's sources plus 'self')`
+            : `${scriptName} ${[...kept, '\'self\''].join(' ')}   (add 'self')`);
+    }
+    if (policy.includes(DYNAMIC))
+        notes.push('Parts of this policy are computed at runtime; init read only its literal parts.');
+    return { additions, notes };
+}
+function lineOf(text, offset) {
+    return text.slice(0, offset).split('\n').length;
+}
+/** The policy text a JS/TS/JSON node spells: a string, a template (expressions as <dynamic>), an array of directive
+ * strings joined, or an object of directives (helmet, SvelteKit kit.csp, nuxt-security). */
+function policyFromNode(node) {
+    const current = unwrapTs(node);
+    if (!current)
+        return null;
+    if (current.type === 'StringLiteral')
+        return current.value;
+    if (current.type === 'TemplateLiteral') {
+        return current.quasis.map((quasi, index) => quasi.value.cooked + (index < current.expressions.length ? DYNAMIC : '')).join('');
+    }
+    if (current.type === 'CallExpression' && current.callee.type === 'MemberExpression') {
+        const method = current.callee.property.name;
+        if (method === 'join' && current.callee.object.type === 'ArrayExpression') {
+            const parts = current.callee.object.elements.map((element) => (element ? policyFromNode(element) : null));
+            return parts.every((part) => part !== null) ? parts.join('; ') : null;
+        }
+        if (method === 'replace' || method === 'trim' || method === 'replaceAll')
+            return policyFromNode(current.callee.object);
+    }
+    if (current.type === 'ObjectExpression') {
+        const directives = [];
+        for (const property of current.properties) {
+            if (property.type !== 'ObjectProperty')
+                return null;
+            const name = keyName(property);
+            if (name === null)
+                return null;
+            const kebab = name.replace(/[A-Z]/g, letter => `-${letter.toLowerCase()}`);
+            if (!/-src(-elem)?$/.test(kebab))
+                continue;
+            const value = unwrapTs(property.value);
+            if (value.type !== 'ArrayExpression') {
+                directives.push(`${kebab} ${DYNAMIC}`);
+                continue;
+            }
+            const sources = value.elements.map((element) => (element?.type === 'StringLiteral' ? element.value : DYNAMIC))
+                // SvelteKit spells keywords without quotes ('self', 'strict-dynamic').
+                .map((source) => (['self', 'none', 'strict-dynamic', 'unsafe-inline', 'unsafe-eval'].includes(source) ? `'${source}'` : source));
+            directives.push(`${kebab} ${sources.join(' ')}`);
+        }
+        return directives.length > 0 ? directives.join('; ') : null;
+    }
+    return null;
+}
+function jsPolicies(text, file, json) {
+    const ast = json ? parseJsonLike(text, file) : parseFile(text, file);
+    if (!ast)
+        return [{ offset: 0, policy: null, reportOnly: false }];
+    const found = [];
+    const seen = new Set();
+    const helmet = json ? null : ast.program.body.flatMap((statement) => (statement.type === 'ImportDeclaration' && statement.source.value === 'helmet'
+        ? statement.specifiers.filter((specifier) => specifier.type === 'ImportDefaultSpecifier').map((specifier) => specifier.local.name) : []))[0] ?? null;
+    walk(json ? ast : ast.program, node => {
+        if (seen.has(node))
+            return false;
+        // { key: 'Content-Security-Policy', value: ... } (Next.js headers(), vercel.json) and 'Content-Security-Policy': ...
+        if (node.type === 'ObjectExpression') {
+            const keyProperty = node.properties.find((property) => property.type === 'ObjectProperty' && keyName(property) === 'key');
+            const valueProperty = node.properties.find((property) => property.type === 'ObjectProperty' && keyName(property) === 'value');
+            const header = keyProperty && unwrapTs(keyProperty.value).type === 'StringLiteral' ? unwrapTs(keyProperty.value).value.toLowerCase() : null;
+            if (header && header.startsWith('content-security-policy') && valueProperty) {
+                found.push({ offset: node.start, policy: resolvePolicy(valueProperty.value, ast), reportOnly: header.endsWith('report-only') });
+                seen.add(valueProperty.value);
+            }
+        }
+        if (node.type === 'ObjectProperty') {
+            const name = keyName(node)?.toLowerCase() ?? '';
+            if (name.startsWith('content-security-policy') || name === 'contentsecuritypolicy' || name === 'csp') {
+                const value = unwrapTs(node.value);
+                if (value.type === 'BooleanLiteral')
+                    return undefined;
+                const directives = value.type === 'ObjectExpression'
+                    ? (value.properties.find((property) => property.type === 'ObjectProperty' && keyName(property) === 'directives')?.value ?? value) : value;
+                const policy = resolvePolicy(directives, ast);
+                // `csp` is a common key for other things; it counts only when it spells a policy.
+                if (name !== 'csp' || (policy !== null && DIRECTIVE.test(policy)))
+                    found.push({ offset: node.start, policy, reportOnly: name.endsWith('report-only') });
+                seen.add(node.value);
+            }
+        }
+        if (node.type === 'CallExpression' && node.callee.type === 'Identifier' && helmet !== null && node.callee.name === helmet) {
+            const options = node.arguments[0] ? unwrapTs(node.arguments[0]) : null;
+            const configured = options?.type === 'ObjectExpression'
+                && options.properties.some((property) => property.type !== 'ObjectProperty' || keyName(property) === 'contentSecurityPolicy');
+            if (!configured)
+                found.push({ offset: node.start, policy: HELMET_DEFAULT, reportOnly: false });
+        }
+        // headers.set('Content-Security-Policy', value) (middleware) and res.setHeader(...).
+        if (node.type === 'CallExpression' && node.arguments.length >= 2 && node.arguments[0].type === 'StringLiteral'
+            && node.arguments[0].value.toLowerCase().startsWith('content-security-policy')) {
+            found.push({ offset: node.start, policy: resolvePolicy(node.arguments[1], ast), reportOnly: node.arguments[0].value.toLowerCase().endsWith('report-only') });
+        }
+        return undefined;
+    });
+    return found;
+}
+/** The initializer of the one variable named `name` anywhere in the module (a policy built in a middleware function),
+ * or null when there is none or more than one. */
+function declaratorInit(ast, name) {
+    const inits = [];
+    walk(ast.program ?? ast, node => {
+        if (node.type === 'VariableDeclarator' && node.id.type === 'Identifier' && node.id.name === name && node.init)
+            inits.push(node.init);
+    });
+    return inits.length === 1 ? inits[0] : null;
+}
+/** A policy expression, through variables (`const cspHeader = \`...\``) and string clean-ups (`.replace(...).trim()`). */
+function resolvePolicy(node, ast, depth = 0) {
+    const current = unwrapTs(node);
+    if (!current || depth > 6)
+        return null;
+    if (current.type === 'Identifier') {
+        const init = declaratorInit(ast, current.name);
+        return init ? resolvePolicy(init, ast, depth + 1) : null;
+    }
+    if (current.type === 'CallExpression' && current.callee.type === 'MemberExpression'
+        && ['replace', 'replaceAll', 'trim'].includes(current.callee.property.name)) {
+        return resolvePolicy(current.callee.object, ast, depth + 1);
+    }
+    // A header's value is its policy whatever it says (`frame-ancestors 'none'` restricts nothing the tag does); a computed
+    // value init cannot read stays null.
+    return policyFromNode(current);
+}
+/** helmet()'s default policy (helmet 5–8), which applies when its options leave contentSecurityPolicy alone. */
+const HELMET_DEFAULT = "default-src 'self'; base-uri 'self'; font-src 'self' https: data:; form-action 'self'; frame-ancestors 'self';"
+    + " img-src 'self' data:; object-src 'none'; script-src 'self'; script-src-attr 'none'; style-src 'self' https: 'unsafe-inline'";
+function metaPolicies(text) {
+    const found = [];
+    const parser = new SAXParser({ sourceCodeLocationInfo: true });
+    parser.on('startTag', token => {
+        if (token.tagName !== 'meta')
+            return;
+        const equiv = token.attrs.find(attribute => attribute.name === 'http-equiv')?.value.toLowerCase();
+        if (equiv !== 'content-security-policy')
+            return;
+        const content = token.attrs.find(attribute => attribute.name === 'content')?.value ?? null;
+        found.push({ offset: token.sourceCodeLocation?.startOffset ?? 0, policy: content && !content.includes('<%') && !content.includes('{{') ? content : null, reportOnly: false });
+    });
+    parser.write(text);
+    parser.end();
+    return found;
+}
+const RUBY_KEYWORDS = { self: '\'self\'', none: '\'none\'', https: 'https:', http: 'http:', data: 'data:', blob: 'blob:',
+    unsafe_inline: '\'unsafe-inline\'', unsafe_eval: '\'unsafe-eval\'', strict_dynamic: '\'strict-dynamic\'', report_sample: '\'report-sample\'' };
+/** Rails' content_security_policy DSL (`policy.script_src :self, :https`), one directive per line. */
+function rubyPolicy(text) {
+    const directives = [];
+    for (const line of text.split('\n')) {
+        const match = /^\s*policy\.([a-z_]+_src(?:_elem)?)\s+(.*)$/.exec(line);
+        if (!match)
+            continue;
+        const sources = match[2].split(',').map(source => source.trim()).map(source => {
+            if (source.startsWith(':'))
+                return RUBY_KEYWORDS[source.slice(1)] ?? DYNAMIC;
+            const quoted = /^["']([^"']+)["']$/.exec(source);
+            return quoted ? quoted[1] : DYNAMIC;
+        });
+        directives.push(`${match[1].replace(/_/g, '-')} ${sources.join(' ')}`);
+    }
+    return directives.length > 0 ? directives.join('; ') : null;
+}
+/** django-csp's CSP_<DIRECTIVE> = (...) settings. */
+function pythonPolicy(text) {
+    const directives = [];
+    for (const match of text.matchAll(/^CSP_([A-Z_]+_SRC(?:_ELEM)?)\s*=\s*[([]([^)\]]*)[)\]]/gm)) {
+        // Python string literals: "'self'" is the CSP keyword 'self'.
+        const sources = [...match[2].matchAll(/"([^"]*)"|'([^']*)'/g)].map(item => item[1] ?? item[2]);
+        directives.push(`${match[1].toLowerCase().replace(/_/g, '-')} ${sources.join(' ')}`);
+    }
+    return directives.length > 0 ? directives.join('; ') : null;
+}
+/** Header-file forms: Netlify/Cloudflare _headers, netlify.toml, nginx add_header, YAML. */
+function textPolicies(text) {
+    const found = [];
+    let offset = 0;
+    for (const line of text.split('\n')) {
+        const at = line.toLowerCase().indexOf('content-security-policy');
+        if (at !== -1 && !line.trimStart().startsWith('#')) {
+            let rest = line.slice(at + 'content-security-policy'.length);
+            const reportOnly = rest.toLowerCase().startsWith('-report-only');
+            if (reportOnly)
+                rest = rest.slice('-report-only'.length);
+            rest = rest.replace(/^["']?\s*[:=]?\s*/, '');
+            const quote = rest[0] === '"' || rest[0] === "'" ? rest[0] : null;
+            const close = quote ? rest.indexOf(quote, 1) : -1;
+            const policy = (quote ? rest.slice(1, close === -1 ? undefined : close) : rest.replace(/;?\s*$/, '')).trim();
+            found.push({ offset: offset + at, policy: DIRECTIVE.test(policy) ? policy : null, reportOnly });
+        }
+        offset += line.length + 1;
+    }
+    return found;
+}
+/** Every Content-Security-Policy the app's directory (and the repository root's deploy configs) keeps, with the
+ * additions each needs. */
+export function findCsp(repoRoot, appDir, nextLoader, origins) {
+    const roots = appDir === repoRoot ? [appDir] : [appDir, repoRoot];
+    const files = new Set();
+    for (const [index, root] of roots.entries()) {
+        const patterns = index === 0 ? SCAN : ['vercel.json', 'netlify.toml', '_headers', 'nginx.conf', '*.conf'];
+        for (const file of fg.sync(patterns, { cwd: root, ignore: IGNORE, deep: index === 0 ? 8 : 1, onlyFiles: true, dot: false }))
+            files.add(join(root, file));
+    }
+    const findings = [];
+    for (const file of [...files].sort()) {
+        try {
+            if (statSync(file).size > MAX_FILE_BYTES)
+                continue;
+        }
+        catch {
+            continue;
+        }
+        const text = readFileSync(file, 'utf8');
+        if (!MARKER.test(text))
+            continue;
+        const extension = extname(file);
+        const where = toPosix(relative(repoRoot, file));
+        let policies;
+        if (['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx'].includes(extension))
+            policies = jsPolicies(text, file, false);
+        else if (extension === '.json')
+            policies = jsPolicies(text, file, true);
+        else if (extension === '.html' || extension === '.erb')
+            policies = metaPolicies(text);
+        // Rails generates its initializer fully commented out: only active `policy.` lines make a policy.
+        else if (extension === '.rb') {
+            policies = rubyPolicy(text) === null ? [] : [{ offset: text.search(/^[ \t]*policy\./m), policy: rubyPolicy(text), reportOnly: /^\s*[^#\n]*report_only\s*=\s*true/m.test(text) }];
+        }
+        else if (extension === '.py')
+            policies = [{ offset: 0, policy: pythonPolicy(text), reportOnly: /CSP_REPORT_ONLY\s*=\s*True/.test(text) }];
+        else
+            policies = textPolicies(text);
+        const unique = policies.filter((item, index) => policies.findIndex(other => other.policy === item.policy && other.reportOnly === item.reportOnly) === index);
+        for (const { offset, policy, reportOnly } of unique) {
+            const analysis = policy === null ? null : analyzePolicy(policy, nextLoader, origins);
+            findings.push({
+                where: `${where}:${lineOf(text, offset)}`,
+                policy,
+                reportOnly,
+                additions: analysis ? analysis.additions : [`connect-src must allow 'self' ${CAPTURE_ORIGIN}, and script-src (or default-src) must allow 'self'`],
+                notes: [...(analysis ? analysis.notes : ['init could not read this policy, which is computed at runtime.']),
+                    ...(reportOnly && (!analysis || analysis.additions.length > 0) ? ['This policy is report-only: it does not block the tag, but would report it.'] : [])],
+            });
+        }
+    }
+    return { findings, looked: `${toPosix(relative(repoRoot, appDir)) || '.'} (headers config, middleware, meta tags, framework CSP options, server configs)` };
+}

package/dist/capture/html.js

@@ -0,0 +1,74 @@
+/**
+ * Adding the tag to an HTML document or a server template (index.html, SvelteKit's app.html, Rails ERB layouts, Django
+ * templates). parse5's tokenizer reads the tags lexically, the way a browser's tokenizer does (comments, script and
+ * style raw text, quoted attributes), without building a tree: a tree builder treats template text in <head>
+ * (`<%= csrf_meta_tags %>`, `{% block %}`, `%sveltekit.head%`) as the end of the head and loses the real </head>.
+ */
+import { SAXParser } from 'parse5-sax-parser';
+import { indentUnit, lineIndent, startsLine } from './js-ast.js';
+import { ourScriptKind } from './tag.js';
+export function scanHtml(text) {
+    const scan = { heads: [], headEnds: [], ours: [] };
+    let open = null;
+    const parser = new SAXParser({ sourceCodeLocationInfo: true });
+    parser.on('startTag', token => {
+        const location = token.sourceCodeLocation;
+        if (!location)
+            return;
+        if (token.tagName === 'head')
+            scan.heads.push({ start: location.startOffset, end: location.endOffset });
+        if (token.tagName === 'script') {
+            const src = token.attrs.find(attribute => attribute.name === 'src')?.value;
+            open = src !== undefined && ourScriptKind(src) !== null ? { start: location.startOffset, end: location.endOffset } : null;
+        }
+    });
+    parser.on('endTag', token => {
+        const location = token.sourceCodeLocation;
+        if (!location)
+            return;
+        if (token.tagName === 'head')
+            scan.headEnds.push({ start: location.startOffset, end: location.endOffset });
+        if (token.tagName === 'script' && open) {
+            scan.ours.push({ start: open.start, end: location.endOffset });
+            open = null;
+        }
+    });
+    parser.write(text);
+    parser.end();
+    return scan;
+}
+/** The range to delete for an element: its whole line when nothing else is on it. */
+function wholeLine(text, range) {
+    const lineStart = text.lastIndexOf('\n', range.start - 1) + 1;
+    const lineEnd = text.indexOf('\n', range.end);
+    const end = lineEnd === -1 ? text.length : lineEnd + 1;
+    return text.slice(lineStart, range.start).trim() === '' && text.slice(range.end, end).trim() === '' ? { start: lineStart, end } : range;
+}
+/**
+ * The document with exactly `tags` as init's scripts, at the end of its one <head>: unchanged when they are already
+ * there; otherwise init's earlier scripts (an older version) are removed and these inserted, so a rerun never duplicates.
+ * A document without exactly one <head> and one </head> is a manual step.
+ */
+export function withHeadTags(text, tags) {
+    const scan = scanHtml(text);
+    if (scan.heads.length !== 1 || scan.headEnds.length !== 1) {
+        return { manual: `it has ${scan.heads.length} <head> and ${scan.headEnds.length} </head> tags, not one of each` };
+    }
+    const current = scan.ours.map(range => text.slice(range.start, range.end));
+    if (current.length === tags.length && current.every((tag, index) => tag === tags[index]))
+        return { content: text };
+    // Remove init's earlier tags, then add these before </head>.
+    let edited = text;
+    const removals = scan.ours.map(range => wholeLine(text, range)).sort((a, b) => b.start - a.start);
+    for (const range of removals)
+        edited = edited.slice(0, range.start) + edited.slice(range.end);
+    const headEnd = scanHtml(edited).headEnds[0];
+    if (!headEnd)
+        return { manual: 'its </head> could not be found again after removing an older tag' };
+    if (startsLine(edited, headEnd.start)) {
+        const indent = lineIndent(edited, headEnd.start) + indentUnit(text);
+        const lineStart = edited.lastIndexOf('\n', headEnd.start - 1) + 1;
+        return { content: edited.slice(0, lineStart) + tags.map(tag => `${indent}${tag}\n`).join('') + edited.slice(lineStart) };
+    }
+    return { content: edited.slice(0, headEnd.start) + tags.join('') + edited.slice(headEnd.start) };
+}