mustflow 2.84.0 → 2.99.0

package/dist/core/secret-risk-scan.js

@@ -0,0 +1,440 @@
+import { createHash } from 'node:crypto';
+import { existsSync, lstatSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import { ensureInside, ensureInsideWithoutSymlinks, readFileInsideWithoutSymlinks } from './safe-filesystem.js';
+export const SECRET_RISK_SCAN_PACK_ID = 'repo';
+export const SECRET_RISK_SCAN_SCRIPT_ID = 'secret-risk-scan';
+export const SECRET_RISK_SCAN_SCRIPT_REF = `${SECRET_RISK_SCAN_PACK_ID}/${SECRET_RISK_SCAN_SCRIPT_ID}`;
+const DEFAULT_MAX_FILE_BYTES = 256 * 1024;
+const DEFAULT_MAX_FILES = 1000;
+const DEFAULT_MAX_FINDINGS = 200;
+const MAX_ISSUES = 50;
+const SCAN_EXTENSIONS = [
+    '.ts',
+    '.tsx',
+    '.mts',
+    '.cts',
+    '.js',
+    '.jsx',
+    '.mjs',
+    '.cjs',
+    '.json',
+    '.toml',
+    '.yml',
+    '.yaml',
+    '.md',
+    '.mdx',
+];
+const SECRET_FILE_NAMES = ['.env', '.env.local', '.env.production', '.env.development', '.dev.vars'];
+const ENV_EXAMPLE_NAMES = [
+    '.env.example',
+    '.env.sample',
+    '.env.template',
+    '.env.defaults',
+    '.env.test.example',
+    '.env.local.example',
+    '.dev.vars.example',
+];
+const IGNORED_DIRECTORIES = [
+    '.git',
+    '.mustflow/cache',
+    '.mustflow/state',
+    'node_modules',
+    'dist',
+    'build',
+    'coverage',
+    '.next',
+    '.turbo',
+];
+const ERROR_CODES = new Set([
+    'secret_risk_path_outside_root',
+    'secret_risk_unreadable_path',
+]);
+function normalizeRelativePath(value) {
+    return value.replace(/\\/gu, '/').replace(/^\.\/+/u, '') || '.';
+}
+function fingerprint(value) {
+    return `sha256:${createHash('sha256').update(value).digest('hex').slice(0, 16)}`;
+}
+function sha256Tagged(value) {
+    return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+function pushIssue(issues, issue) {
+    if (issues.length < MAX_ISSUES) {
+        issues.push(issue);
+    }
+}
+function makeFinding(code, severity, pathValue, message, details = {}) {
+    return { code, severity, path: pathValue, message, ...details };
+}
+function isIgnoredDirectory(relativePath) {
+    const normalized = normalizeRelativePath(relativePath);
+    return IGNORED_DIRECTORIES.some((directory) => normalized === directory || normalized.startsWith(`${directory}/`));
+}
+function isSecretFile(relativePath) {
+    return SECRET_FILE_NAMES.includes(path.basename(relativePath).toLowerCase());
+}
+function isEnvExampleFile(relativePath) {
+    return ENV_EXAMPLE_NAMES.includes(path.basename(relativePath).toLowerCase());
+}
+function surfaceForPath(relativePath) {
+    if (isEnvExampleFile(relativePath)) {
+        return 'example';
+    }
+    const normalized = normalizeRelativePath(relativePath);
+    const extension = path.extname(normalized).toLowerCase();
+    if (!SCAN_EXTENSIONS.includes(extension)) {
+        return null;
+    }
+    if (normalized.startsWith('.github/workflows/') || ['.yml', '.yaml'].includes(extension)) {
+        return 'ci';
+    }
+    if (['.md', '.mdx'].includes(extension)) {
+        return 'docs';
+    }
+    if (['.json', '.toml'].includes(extension) || normalized.startsWith('.mustflow/config/')) {
+        return 'config';
+    }
+    return 'code';
+}
+function normalizeTargetPath(projectRoot, targetPath) {
+    const absolutePath = path.resolve(process.cwd(), targetPath);
+    ensureInside(projectRoot, absolutePath);
+    return {
+        absolutePath,
+        relativePath: normalizeRelativePath(path.relative(projectRoot, absolutePath)),
+    };
+}
+function targetKind(absolutePath) {
+    if (!existsSync(absolutePath)) {
+        return { exists: false, kind: 'missing' };
+    }
+    const stats = lstatSync(absolutePath);
+    if (stats.isFile()) {
+        return { exists: true, kind: 'file' };
+    }
+    if (stats.isDirectory()) {
+        return { exists: true, kind: 'directory' };
+    }
+    return { exists: true, kind: 'other' };
+}
+function addCandidate(candidates, findings, issues, policy, candidate) {
+    if (candidates.has(candidate.relativePath)) {
+        return;
+    }
+    if (candidates.size >= policy.max_files) {
+        if (!findings.some((finding) => finding.code === 'secret_risk_max_files_exceeded')) {
+            const message = `Secret-risk scan matched more than ${policy.max_files} files; remaining files were skipped.`;
+            pushIssue(issues, message);
+            findings.push(makeFinding('secret_risk_max_files_exceeded', 'medium', candidate.relativePath, message));
+        }
+        return;
+    }
+    candidates.set(candidate.relativePath, candidate);
+}
+function collectFilesFromDirectory(projectRoot, absoluteDirectory, candidates, findings, issues, policy) {
+    const relativeDirectory = normalizeRelativePath(path.relative(projectRoot, absoluteDirectory));
+    if (isIgnoredDirectory(relativeDirectory)) {
+        return;
+    }
+    let entries;
+    try {
+        ensureInsideWithoutSymlinks(projectRoot, absoluteDirectory);
+        entries = readdirSync(absoluteDirectory, { withFileTypes: true });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        pushIssue(issues, `${relativeDirectory}: ${message}`);
+        findings.push(makeFinding('secret_risk_unreadable_path', 'high', relativeDirectory, message));
+        return;
+    }
+    for (const entry of entries) {
+        const absoluteEntry = path.join(absoluteDirectory, entry.name);
+        const relativeEntry = normalizeRelativePath(path.relative(projectRoot, absoluteEntry));
+        if (entry.isDirectory()) {
+            collectFilesFromDirectory(projectRoot, absoluteEntry, candidates, findings, issues, policy);
+            continue;
+        }
+        if (!entry.isFile()) {
+            continue;
+        }
+        const surface = surfaceForPath(relativeEntry);
+        if (surface || isSecretFile(relativeEntry)) {
+            addCandidate(candidates, findings, issues, policy, { absolutePath: absoluteEntry, relativePath: relativeEntry, surface: surface ?? 'config' });
+        }
+    }
+}
+function lineNumberAtIndex(text, index) {
+    let line = 1;
+    let offset = 0;
+    while (offset < index) {
+        if (text.charCodeAt(offset) === 10) {
+            line += 1;
+        }
+        offset += 1;
+    }
+    return line;
+}
+function looksLikePlaceholder(value) {
+    const normalized = value.trim().replace(/^['"`]|['"`]$/gu, '').toLowerCase();
+    if (normalized.length === 0) {
+        return true;
+    }
+    if (/^(?:example|sample|dummy|fake|test|changeme|change_me|placeholder|todo|null|undefined|your[_-].*|xxx+|\*+)$/u.test(normalized)) {
+        return true;
+    }
+    return ((normalized.startsWith('<') && normalized.endsWith('>')) ||
+        (normalized.startsWith('${') && normalized.endsWith('}')) ||
+        (normalized.startsWith('{{') && normalized.endsWith('}}')));
+}
+function entropyScore(value) {
+    const normalized = value.trim();
+    if (normalized.length === 0) {
+        return 0;
+    }
+    const counts = new Map();
+    for (const character of normalized) {
+        counts.set(character, (counts.get(character) ?? 0) + 1);
+    }
+    let entropy = 0;
+    for (const count of counts.values()) {
+        const probability = count / normalized.length;
+        entropy -= probability * Math.log2(probability);
+    }
+    return entropy;
+}
+function isSecretLikeName(name) {
+    return /(?:SECRET|TOKEN|PASSWORD|PASS|PRIVATE|CREDENTIAL|API[_-]?KEY|ACCESS[_-]?KEY|AUTH|SESSION|SIGNING|WEBHOOK)/iu.test(name);
+}
+function looksLikeSecretValue(value) {
+    const cleaned = value.trim().replace(/^['"`]|['"`]$/gu, '');
+    if (looksLikePlaceholder(cleaned)) {
+        return false;
+    }
+    if (/^(?:sk-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|xox[baprs]-[A-Za-z0-9-]{20,}|AKIA[0-9A-Z]{16})$/u.test(cleaned)) {
+        return true;
+    }
+    return cleaned.length >= 24 && entropyScore(cleaned) >= 3.5 && /[A-Za-z]/u.test(cleaned) && /\d/u.test(cleaned);
+}
+function addBoundedFinding(findings, issues, policy, finding) {
+    if (findings.length >= policy.max_findings) {
+        if (!findings.some((entry) => entry.code === 'secret_risk_max_findings_exceeded')) {
+            const message = `Secret-risk scan found more than ${policy.max_findings} findings; remaining findings were skipped.`;
+            pushIssue(issues, message);
+            findings.push(makeFinding('secret_risk_max_findings_exceeded', 'medium', finding.path, message));
+        }
+        return;
+    }
+    findings.push(finding);
+}
+function scanPattern(text, candidate, findings, issues, policy, pattern, create) {
+    for (const match of text.matchAll(pattern)) {
+        const line = lineNumberAtIndex(text, match.index ?? 0);
+        const finding = create(match, line);
+        if (finding) {
+            addBoundedFinding(findings, issues, policy, finding);
+        }
+    }
+}
+function scanPrivateKeyBlocks(text, candidate, findings, issues, policy) {
+    scanPattern(text, candidate, findings, issues, policy, /-----BEGIN [A-Z ]*PRIVATE KEY-----/gu, (match, line) => makeFinding('secret_risk_private_key_block', 'critical', candidate.relativePath, 'Private key block marker found.', {
+        line,
+        detector: 'private_key_block',
+        fingerprint: fingerprint(match[0]),
+    }));
+}
+function scanBearerTokens(text, candidate, findings, issues, policy) {
+    scanPattern(text, candidate, findings, issues, policy, /\bBearer\s+(?<value>[A-Za-z0-9._~+/=-]{24,})\b/gu, (match, line) => {
+        const value = match.groups?.value ?? '';
+        if (!looksLikeSecretValue(value)) {
+            return null;
+        }
+        return makeFinding('secret_risk_bearer_token', 'high', candidate.relativePath, 'Bearer token-like value found.', {
+            line,
+            detector: 'bearer_token',
+            fingerprint: fingerprint(value),
+        });
+    });
+}
+function scanProviderTokens(text, candidate, findings, issues, policy) {
+    const pattern = /\b(?<value>sk-[A-Za-z0-9_-]{20,}|ghp_[A-Za-z0-9_]{20,}|github_pat_[A-Za-z0-9_]{20,}|xox[baprs]-[A-Za-z0-9-]{20,}|AKIA[0-9A-Z]{16})\b/gu;
+    scanPattern(text, candidate, findings, issues, policy, pattern, (match, line) => {
+        const value = match.groups?.value ?? '';
+        if (looksLikePlaceholder(value)) {
+            return null;
+        }
+        return makeFinding('secret_risk_provider_token', 'high', candidate.relativePath, 'Provider token-like value found.', {
+            line,
+            detector: 'provider_token',
+            fingerprint: fingerprint(value),
+        });
+    });
+}
+function scanAssignments(text, candidate, findings, issues, policy) {
+    const pattern = /(?:const|let|var|export\s+const|export\s+let|export\s+var)?\s*(?<name>[A-Za-z_][A-Za-z0-9_.-]*(?:SECRET|TOKEN|PASSWORD|PASS|PRIVATE|CREDENTIAL|API[_-]?KEY|ACCESS[_-]?KEY|AUTH|SESSION|SIGNING|WEBHOOK)[A-Za-z0-9_.-]*)\s*[:=]\s*["'`](?<value>[^"'`\r\n]{8,})["'`]/giu;
+    scanPattern(text, candidate, findings, issues, policy, pattern, (match, line) => {
+        const name = match.groups?.name ?? '';
+        const value = match.groups?.value ?? '';
+        if (!isSecretLikeName(name) || !looksLikeSecretValue(value)) {
+            return null;
+        }
+        return makeFinding('secret_risk_generic_assignment', 'high', candidate.relativePath, 'Secret-like assignment found.', {
+            line,
+            detector: 'generic_assignment',
+            fingerprint: fingerprint(`${name}:${value}`),
+        });
+    });
+}
+function scanEnvExample(text, candidate, findings, issues, policy) {
+    const lines = text.split(/\r?\n/u);
+    for (const [index, line] of lines.entries()) {
+        const match = /^\s*(?:export\s+)?(?<name>[A-Za-z_][A-Za-z0-9_.-]*)\s*=\s*(?<value>.+?)\s*$/u.exec(line);
+        if (!match?.groups) {
+            continue;
+        }
+        const { name, value } = match.groups;
+        if (!isSecretLikeName(name) || !looksLikeSecretValue(value)) {
+            continue;
+        }
+        addBoundedFinding(findings, issues, policy, makeFinding('secret_risk_realistic_env_example', 'medium', candidate.relativePath, 'Env example contains a realistic secret-like value.', {
+            line: index + 1,
+            detector: 'realistic_env_example',
+            fingerprint: fingerprint(`${name}:${value}`),
+        }));
+    }
+}
+function inspectCandidate(projectRoot, candidate, policy, findings, issues) {
+    if (isSecretFile(candidate.relativePath)) {
+        addBoundedFinding(findings, issues, policy, makeFinding('secret_risk_secret_file_skipped', 'low', candidate.relativePath, `${candidate.relativePath} was skipped to avoid reading real secret values.`, {
+            detector: 'secret_file_skipped',
+        }));
+        return;
+    }
+    let text;
+    try {
+        text = readFileInsideWithoutSymlinks(projectRoot, candidate.absolutePath, { maxBytes: policy.max_file_bytes }).toString('utf8');
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        pushIssue(issues, `${candidate.relativePath}: ${message}`);
+        findings.push(makeFinding('secret_risk_unreadable_path', 'high', candidate.relativePath, message));
+        return;
+    }
+    scanPrivateKeyBlocks(text, candidate, findings, issues, policy);
+    scanBearerTokens(text, candidate, findings, issues, policy);
+    scanProviderTokens(text, candidate, findings, issues, policy);
+    scanAssignments(text, candidate, findings, issues, policy);
+    if (candidate.surface === 'example') {
+        scanEnvExample(text, candidate, findings, issues, policy);
+    }
+}
+function secretRiskStatus(findings) {
+    if (findings.some((finding) => ERROR_CODES.has(finding.code))) {
+        return 'error';
+    }
+    if (findings.some((finding) => ['medium', 'high', 'critical'].includes(finding.severity))) {
+        return 'failed';
+    }
+    return 'passed';
+}
+function summarizeSecretRisk(targets, fileCount, findings) {
+    return {
+        target_count: targets.length,
+        file_count: fileCount,
+        finding_count: findings.length,
+        skipped_secret_file_count: findings.filter((finding) => finding.code === 'secret_risk_secret_file_skipped').length,
+        high_or_critical_count: findings.filter((finding) => ['high', 'critical'].includes(finding.severity)).length,
+    };
+}
+function createInputHash(policy, targets, findings, issues) {
+    return sha256Tagged(JSON.stringify({
+        policy,
+        targets,
+        findings: findings.map((finding) => ({
+            code: finding.code,
+            path: finding.path,
+            line: finding.line,
+            detector: finding.detector,
+            fingerprint: finding.fingerprint,
+        })),
+        issues,
+    }));
+}
+export function inspectSecretRiskScan(projectRoot, options = {}) {
+    const root = path.resolve(projectRoot);
+    const policy = {
+        max_file_bytes: options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES,
+        max_files: options.maxFiles ?? DEFAULT_MAX_FILES,
+        max_findings: options.maxFindings ?? DEFAULT_MAX_FINDINGS,
+        extensions: [...SCAN_EXTENSIONS],
+        skipped_secret_names: [...SECRET_FILE_NAMES],
+        ignored_directories: [...IGNORED_DIRECTORIES],
+    };
+    const targetInputs = options.paths && options.paths.length > 0 ? options.paths : ['.'];
+    const targets = [];
+    const candidates = new Map();
+    const findings = [];
+    const issues = [];
+    for (const targetPath of targetInputs) {
+        let absolutePath;
+        let relativePath;
+        try {
+            const normalized = normalizeTargetPath(root, targetPath);
+            absolutePath = normalized.absolutePath;
+            relativePath = normalized.relativePath;
+            ensureInsideWithoutSymlinks(root, absolutePath, { allowMissingLeaf: true });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            pushIssue(issues, message);
+            targets.push({ input: targetPath, path: targetPath, exists: null, kind: 'unknown' });
+            findings.push(makeFinding('secret_risk_path_outside_root', 'high', targetPath, message));
+            continue;
+        }
+        let existence;
+        try {
+            existence = targetKind(absolutePath);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            pushIssue(issues, `${relativePath}: ${message}`);
+            targets.push({ input: targetPath, path: relativePath, exists: null, kind: 'unknown' });
+            findings.push(makeFinding('secret_risk_unreadable_path', 'high', relativePath, message));
+            continue;
+        }
+        targets.push({ input: targetPath, path: relativePath, exists: existence.exists, kind: existence.kind });
+        if (existence.kind === 'file') {
+            const surface = surfaceForPath(relativePath);
+            if (surface || isSecretFile(relativePath)) {
+                addCandidate(candidates, findings, issues, policy, { absolutePath, relativePath, surface: surface ?? 'config' });
+            }
+        }
+        else if (existence.kind === 'directory') {
+            collectFilesFromDirectory(root, absolutePath, candidates, findings, issues, policy);
+        }
+    }
+    for (const candidate of candidates.values()) {
+        inspectCandidate(root, candidate, policy, findings, issues);
+    }
+    const status = secretRiskStatus(findings);
+    const truncated = findings.some((finding) => ['secret_risk_max_files_exceeded', 'secret_risk_max_findings_exceeded'].includes(finding.code));
+    const summary = summarizeSecretRisk(targets, candidates.size, findings);
+    return {
+        schema_version: '1',
+        command: 'script-pack',
+        pack_id: SECRET_RISK_SCAN_PACK_ID,
+        script_id: SECRET_RISK_SCAN_SCRIPT_ID,
+        script_ref: SECRET_RISK_SCAN_SCRIPT_REF,
+        action: 'scan',
+        status,
+        ok: status === 'passed',
+        mustflow_root: root,
+        policy,
+        input_hash: createInputHash(policy, targets, findings, issues),
+        targets,
+        summary,
+        truncated,
+        findings,
+        issues,
+    };
+}