hackmyagent 0.16.5 → 0.16.7

package/dist/nanomind-core/compiler/source-code-preprocessor.js

@@ -0,0 +1,656 @@
+"use strict";
+/**
+ * Source Code Preprocessor
+ *
+ * The config-oriented detectors in the semantic compiler
+ * (`extractDataAccessPatterns`, `mapRiskSurfaces`) were designed for
+ * skills, agent configs, and system prompts, where the entire content is
+ * semantically meaningful. Running them against source code produces
+ * reflexive false positives: any file whose *purpose* is to scan for
+ * attack patterns (a credential regex, an `eval(` pattern, a typosquatting
+ * detector) contains the exact strings those detectors look for.
+ *
+ * This preprocessor produces a "stripped view" of source code with
+ * comments, import statements, and string literals replaced by whitespace.
+ * The stripped view preserves byte offsets so any index-based analysis
+ * downstream keeps working. The original content is left untouched for
+ * callers that need it (evidence extraction, capability parsing, etc.).
+ *
+ * What survives the strip:
+ *   - Identifiers (variable, function, and type names)
+ *   - Control flow and operators
+ *   - Struct tags and field names (outside of string literals)
+ *
+ * What is removed:
+ *   - `//`, `#`, `--` line comments
+ *   - `/* ... *\/`, `"""..."""`, `'''...'''` block comments
+ *   - `"..."`, `'...'`, `` `...` ``, `r"..."`, etc. string literals
+ *   - `import (...)` blocks and `import "..."` / `from X import Y` lines
+ *
+ * The net effect: code that references credential/eval/RCE patterns
+ * defensively (scanners, allowlists, documentation strings) no longer
+ * trips reflexive regex matches, while code that actually *does* those
+ * things (e.g. `exec(userInput)` where `exec` is a symbol call, not a
+ * string) still surfaces.
+ *
+ * Security note: stripping string literals trades off a narrow detection
+ * case — a hardcoded secret embedded in a string literal whose *exact*
+ * bytes match an AWS/Anthropic/GitHub key format — for eliminating the
+ * dominant false positive mode on source code. Canonical API key formats
+ * are still detected upstream of this path: `classifyArtifactType` routes
+ * files matching `sk-ant-`, `AKIA...`, `ghp_...`, or PEM headers into the
+ * `credential_file` artifact type before source code preprocessing runs,
+ * so those cases are preserved.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSourceLanguage = detectSourceLanguage;
+exports.buildAnalysisView = buildAnalysisView;
+exports.stripSourceCode = stripSourceCode;
+/** Detect source language from a file path. Returns 'unknown' for unsupported extensions. */
+function detectSourceLanguage(path) {
+    if (!path)
+        return 'unknown';
+    const lower = path.toLowerCase();
+    if (lower.endsWith('.go'))
+        return 'go';
+    if (lower.endsWith('.ts') || lower.endsWith('.tsx'))
+        return 'typescript';
+    if (lower.endsWith('.js') ||
+        lower.endsWith('.jsx') ||
+        lower.endsWith('.mjs') ||
+        lower.endsWith('.cjs')) {
+        return 'javascript';
+    }
+    if (lower.endsWith('.py') || lower.endsWith('.pyi'))
+        return 'python';
+    if (lower.endsWith('.rs'))
+        return 'rust';
+    if (lower.endsWith('.java'))
+        return 'java';
+    if (lower.endsWith('.rb'))
+        return 'ruby';
+    return 'unknown';
+}
+// ============================================================================
+// Public API
+// ============================================================================
+/**
+ * Build an analysis view of content for use with config-oriented detectors.
+ *
+ * For source_code artifacts, strips comments, imports, and string literals.
+ * For all other artifact types, returns the original content unchanged.
+ *
+ * The stripped view preserves byte offsets: removed bytes are replaced with
+ * spaces (or newlines inside multi-line regions), so any downstream code
+ * that uses absolute indices remains valid.
+ */
+function buildAnalysisView(content, artifactType, path) {
+    if (artifactType !== 'source_code') {
+        return content;
+    }
+    const language = detectSourceLanguage(path);
+    if (language === 'unknown') {
+        return content;
+    }
+    return stripSourceCode(content, language);
+}
+/**
+ * Strip comments, imports, and string literals from source code.
+ * Exposed for tests. Callers should usually use `buildAnalysisView`.
+ */
+function stripSourceCode(content, language) {
+    switch (language) {
+        case 'go':
+            return stripGo(content);
+        case 'typescript':
+        case 'javascript':
+            return stripJsLike(content);
+        case 'python':
+            return stripPython(content);
+        case 'rust':
+            return stripRust(content);
+        case 'java':
+            return stripJava(content);
+        case 'ruby':
+            return stripRuby(content);
+        default:
+            return content;
+    }
+}
+// ============================================================================
+// Language-specific strippers
+// ============================================================================
+//
+// Each stripper walks the content once and produces a same-length buffer
+// where comments, imports, and string literals are replaced with spaces
+// (newlines preserved so line numbers are stable).
+//
+// These are hand-rolled tokenizers. They are not language-complete parsers,
+// but they are strict enough for our purpose: stripping signal that the
+// config-oriented regex detectors should not see. A cosmetic miscount (e.g.
+// an unterminated string) may leave a trailing region un-stripped, which is
+// acceptable (worst case, we leave an FP in place; we never silence a real
+// finding outside source code files).
+function blank(n, newlines) {
+    if (newlines === 0)
+        return ' '.repeat(n);
+    return ' '.repeat(n - newlines) + '\n'.repeat(newlines);
+}
+function countNewlines(s) {
+    let count = 0;
+    for (let i = 0; i < s.length; i++)
+        if (s.charCodeAt(i) === 10)
+            count++;
+    return count;
+}
+// ----- Go -----
+function stripGo(src) {
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    // Track start-of-line for import detection
+    while (i < n) {
+        const ch = src[i];
+        const next = src[i + 1];
+        // Line comment
+        if (ch === '/' && next === '/') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        // Block comment
+        if (ch === '/' && next === '*') {
+            const end = src.indexOf('*/', i + 2);
+            const stop = end === -1 ? n : end + 2;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        // Interpreted string literal
+        if (ch === '"') {
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === '"' || c === '\n') {
+                    i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Raw string literal
+        if (ch === '`') {
+            const start = i;
+            i++;
+            const end = src.indexOf('`', i);
+            i = end === -1 ? n : end + 1;
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Rune literal (skip to avoid confusing quoting)
+        if (ch === "'") {
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === "'" || c === '\n') {
+                    i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Import block / import statement at start of a line
+        if (ch === 'i' && isImportAtLineStart(src, i, 'go')) {
+            const consumed = consumeGoImport(src, i);
+            out.push(blank(consumed - i, countNewlines(src.slice(i, consumed))));
+            i = consumed;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+function isImportAtLineStart(src, i, lang) {
+    // Check that `import` keyword starts this line (allowing leading whitespace)
+    let j = i - 1;
+    while (j >= 0 && (src[j] === ' ' || src[j] === '\t'))
+        j--;
+    if (j >= 0 && src[j] !== '\n')
+        return false;
+    // Check keyword
+    const keyword = lang === 'java' ? 'import ' : 'import';
+    if (src.slice(i, i + keyword.length) !== keyword)
+        return false;
+    // After `import`, must be whitespace or `(` (Go) or `{` / identifier (rust)
+    const after = src[i + keyword.length];
+    return after === ' ' || after === '\t' || after === '(' || after === '\n';
+}
+function consumeGoImport(src, i) {
+    // `import` keyword, optional space, then either `(...)` block or `"..."` single
+    let j = i + 'import'.length;
+    while (j < src.length && (src[j] === ' ' || src[j] === '\t'))
+        j++;
+    if (src[j] === '(') {
+        const end = src.indexOf(')', j);
+        return end === -1 ? src.length : end + 1;
+    }
+    // Single-line: consume to end of line
+    const end = src.indexOf('\n', j);
+    return end === -1 ? src.length : end;
+}
+// ----- JavaScript / TypeScript -----
+function stripJsLike(src) {
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    while (i < n) {
+        const ch = src[i];
+        const next = src[i + 1];
+        // Line comment
+        if (ch === '/' && next === '/') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        // Block comment
+        if (ch === '/' && next === '*') {
+            const end = src.indexOf('*/', i + 2);
+            const stop = end === -1 ? n : end + 2;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        // String literals
+        if (ch === '"' || ch === "'") {
+            const quote = ch;
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === quote || c === '\n') {
+                    if (c === quote)
+                        i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Template literal
+        if (ch === '`') {
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                // Nested ${...} interpolation — leave contents visible; rewind
+                if (c === '$' && src[i + 1] === '{') {
+                    // Find matching brace
+                    let depth = 1;
+                    let k = i + 2;
+                    while (k < n && depth > 0) {
+                        if (src[k] === '{')
+                            depth++;
+                        else if (src[k] === '}')
+                            depth--;
+                        if (depth > 0)
+                            k++;
+                    }
+                    // Blank only the literal portion up to `${`
+                    const segmentEnd = i;
+                    out.push(blank(segmentEnd - start, countNewlines(src.slice(start, segmentEnd))));
+                    // Preserve interpolation content verbatim for analysis
+                    out.push(src.slice(i, k + 1));
+                    i = k + 1;
+                    // Continue template from after `}`
+                    while (i < n) {
+                        const c2 = src[i];
+                        if (c2 === '\\' && i + 1 < n) {
+                            i += 2;
+                            continue;
+                        }
+                        if (c2 === '`') {
+                            i++;
+                            return out.concat(stripJsLike(src.slice(i))).join('');
+                        }
+                        if (c2 === '$' && src[i + 1] === '{')
+                            break;
+                        i++;
+                    }
+                    // If we fall out naturally (no more interpolation, no closing backtick)
+                    continue;
+                }
+                if (c === '`') {
+                    i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Import statement at start of line
+        if (ch === 'i' && isImportAtJsLineStart(src, i)) {
+            const end = consumeJsImport(src, i);
+            out.push(blank(end - i, countNewlines(src.slice(i, end))));
+            i = end;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+function isImportAtJsLineStart(src, i) {
+    let j = i - 1;
+    while (j >= 0 && (src[j] === ' ' || src[j] === '\t'))
+        j--;
+    if (j >= 0 && src[j] !== '\n')
+        return false;
+    if (src.slice(i, i + 7) === 'import ' || src.slice(i, i + 7) === 'import{')
+        return true;
+    if (src.slice(i, i + 7) === 'import*')
+        return true;
+    return false;
+}
+function consumeJsImport(src, i) {
+    // Consume until the terminating semicolon or newline where the statement ends
+    // (accounting for multi-line import blocks with braces)
+    let j = i;
+    let brace = 0;
+    while (j < src.length) {
+        const c = src[j];
+        if (c === '{')
+            brace++;
+        else if (c === '}')
+            brace--;
+        else if (c === ';' && brace === 0)
+            return j + 1;
+        else if (c === '\n' && brace === 0) {
+            // If next non-space char is not a continuation, end here
+            let k = j + 1;
+            while (k < src.length && (src[k] === ' ' || src[k] === '\t'))
+                k++;
+            if (k === src.length || src[k] !== 'f' /* from */) {
+                // Heuristic: end of import
+                return j;
+            }
+        }
+        j++;
+    }
+    return j;
+}
+// ----- Python -----
+function stripPython(src) {
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    while (i < n) {
+        const ch = src[i];
+        // Comment
+        if (ch === '#') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        // Triple-quoted string
+        if ((ch === '"' || ch === "'") &&
+            src[i + 1] === ch &&
+            src[i + 2] === ch) {
+            const triple = ch + ch + ch;
+            const start = i;
+            i += 3;
+            const end = src.indexOf(triple, i);
+            i = end === -1 ? n : end + 3;
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Regular string (allow prefix like r, b, f)
+        if (ch === '"' || ch === "'") {
+            const quote = ch;
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === quote || c === '\n') {
+                    if (c === quote)
+                        i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // Import statement at start of line
+        if ((ch === 'i' || ch === 'f') && isImportAtPyLineStart(src, i)) {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+function isImportAtPyLineStart(src, i) {
+    let j = i - 1;
+    while (j >= 0 && (src[j] === ' ' || src[j] === '\t'))
+        j--;
+    if (j >= 0 && src[j] !== '\n')
+        return false;
+    if (src.slice(i, i + 7) === 'import ')
+        return true;
+    if (src.slice(i, i + 5) === 'from ')
+        return true;
+    return false;
+}
+// ----- Rust -----
+function stripRust(src) {
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    while (i < n) {
+        const ch = src[i];
+        const next = src[i + 1];
+        if (ch === '/' && next === '/') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        if (ch === '/' && next === '*') {
+            const end = src.indexOf('*/', i + 2);
+            const stop = end === -1 ? n : end + 2;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        if (ch === '"') {
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === '"') {
+                    i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // `use crate::foo;` and `extern crate foo;`
+        if ((ch === 'u' || ch === 'e') && isRustUseAtLineStart(src, i)) {
+            const end = src.indexOf(';', i);
+            const stop = end === -1 ? n : end + 1;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+function isRustUseAtLineStart(src, i) {
+    let j = i - 1;
+    while (j >= 0 && (src[j] === ' ' || src[j] === '\t'))
+        j--;
+    if (j >= 0 && src[j] !== '\n')
+        return false;
+    return src.slice(i, i + 4) === 'use ' || src.slice(i, i + 13) === 'extern crate ';
+}
+// ----- Java -----
+function stripJava(src) {
+    // Java syntax overlaps heavily with the JS-like stripper for comments and strings;
+    // imports start with `import `. Reuse JS-like with a small tweak.
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    while (i < n) {
+        const ch = src[i];
+        const next = src[i + 1];
+        if (ch === '/' && next === '/') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        if (ch === '/' && next === '*') {
+            const end = src.indexOf('*/', i + 2);
+            const stop = end === -1 ? n : end + 2;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        if (ch === '"') {
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === '"' || c === '\n') {
+                    if (c === '"')
+                        i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        if (ch === 'i' && isImportAtLineStart(src, i, 'java')) {
+            const end = src.indexOf(';', i);
+            const stop = end === -1 ? n : end + 1;
+            out.push(blank(stop - i, countNewlines(src.slice(i, stop))));
+            i = stop;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+// ----- Ruby -----
+function stripRuby(src) {
+    const out = [];
+    let i = 0;
+    const n = src.length;
+    while (i < n) {
+        const ch = src[i];
+        if (ch === '#') {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            const quote = ch;
+            const start = i;
+            i++;
+            while (i < n) {
+                const c = src[i];
+                if (c === '\\' && i + 1 < n) {
+                    i += 2;
+                    continue;
+                }
+                if (c === quote || c === '\n') {
+                    if (c === quote)
+                        i++;
+                    break;
+                }
+                i++;
+            }
+            out.push(blank(i - start, countNewlines(src.slice(start, i))));
+            continue;
+        }
+        // `require`, `require_relative`, `load` at start of line
+        if (ch === 'r' && isRubyRequireAtLineStart(src, i)) {
+            const end = src.indexOf('\n', i);
+            const stop = end === -1 ? n : end;
+            out.push(blank(stop - i, 0));
+            i = stop;
+            continue;
+        }
+        out.push(ch);
+        i++;
+    }
+    return out.join('');
+}
+function isRubyRequireAtLineStart(src, i) {
+    let j = i - 1;
+    while (j >= 0 && (src[j] === ' ' || src[j] === '\t'))
+        j--;
+    if (j >= 0 && src[j] !== '\n')
+        return false;
+    return (src.slice(i, i + 8) === 'require ' ||
+        src.slice(i, i + 17) === 'require_relative ' ||
+        src.slice(i, i + 5) === 'load ');
+}
+//# sourceMappingURL=source-code-preprocessor.js.map