clckernel 1.2.19 → 2.0.0

package/tools/postgres_guard.py

@@ -9,33 +9,14 @@ from __future__ import annotations
 import ast
 import json
 import os
-import subprocess
 import sys
-from pathlib import Path
 
-
-def _get_changed_files(target_dir: str) -> list[str] | None:
-    """Return list of changed Python files from git diff, or None on failure."""
-    for ref in ("HEAD~1", "--cached"):
-        try:
-            args = ["git", "diff", "--name-only"]
-            if ref == "--cached":
-                args.append("--cached")
-            else:
-                args.append(ref)
-            out = subprocess.check_output(
-                args, cwd=target_dir,
-                stderr=subprocess.DEVNULL, text=True,
-            )
-            files = [
-                f.strip() for f in out.splitlines()
-                if f.strip().endswith('.py')
-            ]
-            if files:
-                return files
-        except (subprocess.CalledProcessError, FileNotFoundError):
-            continue
-    return None
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
 
 
 class PostgresVisitor(ast.NodeVisitor):
@@ -85,7 +66,8 @@ class PostgresVisitor(ast.NodeVisitor):
 
     def visit_Call(self, node: ast.Call):
         # Check for cursor.execute(), connection.execute(), etc.
-        if self._is_sql_method(node):
+        # The method name lives on the callee (node.func), not on the Call node.
+        if self._is_sql_method(node.func):
             # Check for raw SQL with string formatting (f-string, %, .format())
             if self._has_formatted_sql(node):
                 self._add_violation(
@@ -138,8 +120,10 @@ def check(target_dir: str | None = None) -> dict:
     """
     dir_path = target_dir or os.getcwd()
 
-    files = _get_changed_files(dir_path)
-    if files is None:
+    # The snapshot being verified: the index, or HEAD when nothing is staged.
+    changes = _changes.changed_files(dir_path, extensions=[".py"])
+    mode, files = changes["mode"], changes["files"]
+    if mode is None:
         return {
             "exit_code": 0, "skipped": True, "name": "postgres_guard",
             "data": {"findings": [], "message": "No git diff available"},
@@ -154,17 +138,12 @@ def check(target_dir: str | None = None) -> dict:
     all_violations: list[dict] = []
 
     for filepath in files:
-        full_path = Path(dir_path) / filepath
-        if not full_path.exists():
-            continue
-
-        try:
-            content = full_path.read_text(encoding="utf-8", errors="replace")
-        except (OSError, PermissionError):
+        content = _changes.read_file(dir_path, filepath, mode)
+        if content is None:
             continue
 
         try:
-            tree = ast.parse(content, filename=str(full_path))
+            tree = ast.parse(content, filename=filepath)
         except SyntaxError:
             continue
 

package/tools/redis_guard.py

@@ -9,33 +9,14 @@ from __future__ import annotations
 import ast
 import json
 import os
-import subprocess
 import sys
-from pathlib import Path
 
-
-def _get_changed_files(target_dir: str) -> list[str] | None:
-    """Return list of changed Python files from git diff, or None on failure."""
-    for ref in ("HEAD~1", "--cached"):
-        try:
-            args = ["git", "diff", "--name-only"]
-            if ref == "--cached":
-                args.append("--cached")
-            else:
-                args.append(ref)
-            out = subprocess.check_output(
-                args, cwd=target_dir,
-                stderr=subprocess.DEVNULL, text=True,
-            )
-            files = [
-                f.strip() for f in out.splitlines()
-                if f.strip().endswith('.py')
-            ]
-            if files:
-                return files
-        except (subprocess.CalledProcessError, FileNotFoundError):
-            continue
-    return None
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
 
 
 class RedisVisitor(ast.NodeVisitor):
@@ -143,8 +124,10 @@ def check(target_dir: str | None = None) -> dict:
     """
     dir_path = target_dir or os.getcwd()
 
-    files = _get_changed_files(dir_path)
-    if files is None:
+    # The snapshot being verified: the index, or HEAD when nothing is staged.
+    changes = _changes.changed_files(dir_path, extensions=[".py"])
+    mode, files = changes["mode"], changes["files"]
+    if mode is None:
         return {
             "exit_code": 0, "skipped": True, "name": "redis_guard",
             "data": {"findings": [], "message": "No git diff available"},
@@ -159,17 +142,12 @@ def check(target_dir: str | None = None) -> dict:
     all_violations: list[dict] = []
 
     for filepath in files:
-        full_path = Path(dir_path) / filepath
-        if not full_path.exists():
-            continue
-
-        try:
-            content = full_path.read_text(encoding="utf-8", errors="replace")
-        except (OSError, PermissionError):
+        content = _changes.read_file(dir_path, filepath, mode)
+        if content is None:
             continue
 
         try:
-            tree = ast.parse(content, filename=str(full_path))
+            tree = ast.parse(content, filename=filepath)
         except SyntaxError:
             continue
 

package/tools/scan_secrets.js

@@ -4,51 +4,146 @@
  * Detects hardcoded secrets, API keys, tokens, and private keys in staged/changed files.
  * Self-contained: uses only Node.js built-ins.
  *
+ * Rules (sdds/secret-patterns) are tried in order; the first match labels the
+ * line (one finding per line). They mirror RULES in scan_secrets.py exactly.
+ * The two generic rules skip placeholder values (see isPlaceholder):
+ *   - Quoted Key Secret: a quoted JSON/YAML key ending in a secret word, with a
+ *     quoted value of 8+ non-space characters holding a letter and a digit;
+ *   - Dotenv Secret: in `.env`, `.env.*` and `*.env` files (not `.env.example`,
+ *     `.env.sample`, `.env.template`), `NAME=value` where a `_` segment of NAME
+ *     is a secret word and the value is not numeric-only or a boolean/null word.
+ *
  * @param {string} targetDir - Directory to scan (defaults to cwd)
  * @returns {{ exitCode: number, skipped: boolean, name: string, data: object }}
  */
-const check = async function scanSecrets(targetDir) {
-  const fs = require('fs');
-  const path = require('path');
-  const { execSync } = require('child_process');
+const path = require('path');
+const { changedFiles, readFile } = require('./_changes');
 
-  const dir = targetDir || process.cwd();
+const SECRET_KEY_WORDS = 'api[_-]?key|apikey|secret[_-]?key|api[_-]?secret|client[_-]?secret|secret|'
+  + 'access[_-]?token|auth[_-]?token|refresh[_-]?token|token|password|passwd|private[_-]?key';
+const DOTENV_SECRET_NAME = /(?:^|_)(?:API_?KEY|KEY|SECRET|TOKEN|PASSWORD|PASSWD|PWD|PASS|CREDENTIALS?|DSN|PRIVATE)(?:_|$)/i;
+const PLACEHOLDER_WORDS = ['xxx', 'your', 'changeme', 'change_me', 'change-me', 'placeholder', 'example',
+  'dummy', 'replace', 'insert', 'redacted', 'fake', 'sample', 'todo', '...', '***'];
+const NON_SECRET_WORDS = ['true', 'false', 'yes', 'no', 'on', 'off', 'null', 'none'];
+const TEMPLATE_SEGMENTS = ['example', 'sample', 'template'];
+
+/** True for values that are obviously not a real secret. */
+function isPlaceholder(value) {
+  const v = value.toLowerCase();
+  if (v.startsWith('$') || v.includes('${') || v.includes('$(') || v.includes('{{')) return true;
+  if (v.includes('<') && v.includes('>')) return true;
+  if (/^(.)\1*$/.test(v)) return true;
+  return PLACEHOLDER_WORDS.some((w) => v.includes(w));
+}
+
+/** The value of a dotenv assignment: quotes removed, or an unquoted value without its ` #` comment. */
+function dotenvValue(raw) {
+  const quoted = /^(["'])([^\n]*?)\1/.exec(raw);
+  if (quoted) return quoted[2];
+  const comment = raw.search(/\s#/);
+  return (comment === -1 ? raw : raw.slice(0, comment)).trim();
+}
+
+/** `.env`, `.env.<suffix>` or `<name>.env`, unless a dot segment marks it as a template. */
+function isDotenvFile(filename) {
+  const base = path.basename(filename || '');
+  if (!(base === '.env' || base.startsWith('.env.') || (base.endsWith('.env') && base.length > 4))) return false;
+  return !base.toLowerCase().split('.').some((seg) => TEMPLATE_SEGMENTS.includes(seg));
+}
+
+const RULES = [
+  { source: '-----BEGIN (?:RSA |EC )?PRIVATE KEY-----', flags: '', label: 'Private Key' },
+  { source: 'AKIA[0-9A-Z]{16}', flags: '', label: 'AWS Access Key' },
+  { source: '(?:api[_-]?key|apikey|secret[_-]?key|api[_-]?secret)\\s*[=:]\\s*[\'"][A-Za-z0-9_\\-]{16,}[\'"]', flags: 'i', label: 'API Key' },
+  { source: '(?:password|passwd|pwd)\\s*[=:]\\s*[\'"][^\'"]{6,}[\'"]', flags: 'i', label: 'Hardcoded Password' },
+  { source: '(?:secret|token|auth[_-]?token|bearer)\\s*[=:]\\s*[\'"][A-Za-z0-9_\\-\\.]{16,}[\'"]', flags: 'i', label: 'Secret/Token' },
+  { source: 'ghp_[A-Za-z0-9]{36}', flags: '', label: 'GitHub Personal Access Token' },
+  { source: 'sk-[A-Za-z0-9]{20,}', flags: '', label: 'OpenAI API Key' },
+  { source: '(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}', flags: '', label: 'Stripe API Key' },
+  { source: 'eyJ[A-Za-z0-9_-]+\\.eyJ[A-Za-z0-9_-]+\\.[A-Za-z0-9_-]+', flags: '', label: 'JWT Token' },
+  { source: '(?:SUPABASE|NEXT_PUBLIC|VITE)_\\w*_KEY\\s*=\\s*[\'"][^\'"]{8,}[\'"]', flags: 'i', label: 'Framework Secret Key' },
+  { source: 'sk-ant-[A-Za-z0-9_-]{20,}', flags: '', label: 'Anthropic API Key' },
+  { source: 'sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{20,}', flags: '', label: 'OpenAI Project Key' },
+  { source: 'AIza[0-9A-Za-z_-]{35}', flags: '', label: 'Google API Key' },
+  { source: 'xox[abprs]-[0-9]+-[0-9A-Za-z-]{8,}', flags: '', label: 'Slack Token' },
+  { source: 'gh[ousr]_[A-Za-z0-9]{36}', flags: '', label: 'GitHub Token' },
+  { source: 'github_pat_[A-Za-z0-9_]{40,}', flags: '', label: 'GitHub Fine-Grained PAT' },
+  {
+    source: `(["'])[A-Za-z0-9_.-]*(?:${SECRET_KEY_WORDS})\\1\\s*:\\s*(["'])([^"'\\s]{8,})\\2`,
+    flags: 'i',
+    label: 'Quoted Key Secret',
+    accept: (m) => /[A-Za-z]/.test(m[3]) && /[0-9]/.test(m[3]) && !isPlaceholder(m[3]),
+  },
+  {
+    source: '^\\s*(?:export\\s+)?([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*([^\\n]*)$',
+    flags: '',
+    label: 'Dotenv Secret',
+    dotenvOnly: true,
+    accept: (m) => {
+      if (!DOTENV_SECRET_NAME.test(m[1])) return false;
+      const value = dotenvValue(m[2].trim());
+      if (!value || /^[0-9]+$/.test(value) || NON_SECRET_WORDS.includes(value.toLowerCase())) return false;
+      return !isPlaceholder(value);
+    },
+  },
+].map((r) => ({ ...r, regex: new RegExp(r.source, r.flags) }));
+
+const REDACTED = '…[REDACTED]';
+
+/**
+ * The line with every match of every applicable rule cut to its first 4
+ * characters plus a marker, trimmed and capped at 120 characters. Filters are
+ * ignored here: over-redaction is safe. The full matched value never leaves the guard.
+ * @param {string} line
+ * @param {{ regex: RegExp }[]} rules
+ * @returns {string}
+ */
+function redact(line, rules) {
+  let out = line;
+  for (const { regex } of rules) {
+    const global = new RegExp(regex.source, regex.flags.includes('g') ? regex.flags : regex.flags + 'g');
+    out = out.replace(global, (m) => m.slice(0, 4) + REDACTED);
+  }
+  return out.trim().substring(0, 120);
+}
 
-  // Collect changed files via git, fall back to package.json check
-  let files;
-  try {
-    const out = execSync('git diff --name-only HEAD~1', {
-      cwd: dir, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe']
-    });
-    files = out.split('\n').filter(f => f.trim());
-  } catch {
-    try {
-      const out = execSync('git diff --name-only --cached', {
-        cwd: dir, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe']
-      });
-      files = out.split('\n').filter(f => f.trim());
-    } catch {
-      // No git history — skip gracefully
-      return { exitCode: 0, skipped: true, name: 'scan_secrets', data: { findings: {}, message: 'No git diff available' } };
+/**
+ * Pure matcher: the findings for `text` as the content of `filename`.
+ * @param {string} text
+ * @param {string} filename - repo-relative path; decides whether dotenv rules apply
+ * @returns {{ lineNo: number, col: number, label: string, snippet: string }[]}
+ */
+function findSecrets(text, filename) {
+  const dotenv = isDotenvFile(filename);
+  const rules = RULES.filter((r) => dotenv || !r.dotenvOnly);
+  const findings = [];
+  const lines = String(text).split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    for (const rule of rules) {
+      const match = rule.regex.exec(line);
+      if (match && (!rule.accept || rule.accept(match))) {
+        findings.push({ lineNo: i + 1, col: match.index, label: rule.label, snippet: redact(line, rules) });
+        break; // One finding per line
+      }
     }
   }
+  return findings;
+}
+
+const check = async function scanSecrets(targetDir) {
+  const dir = targetDir || process.cwd();
+
+  // The snapshot being verified: the index, or HEAD when nothing is staged.
+  const { mode, files } = changedFiles(dir);
+  if (mode === null) {
+    return { exitCode: 0, skipped: true, name: 'scan_secrets', data: { findings: {}, message: 'No git diff available' } };
+  }
 
   if (files.length === 0) {
     return { exitCode: 0, skipped: false, name: 'scan_secrets', data: { findings: {}, message: 'No changed files to scan' } };
   }
 
-  const patterns = [
-    { regex: /-----BEGIN (?:RSA |EC )?PRIVATE KEY-----/, label: 'Private Key' },
-    { regex: /AKIA[0-9A-Z]{16}/, label: 'AWS Access Key' },
-    { regex: /(?:api[_-]?key|apikey|secret[_-]?key|api[_-]?secret)\s*[=:]\s*['"][A-Za-z0-9_\-]{16,}['"]/i, label: 'API Key' },
-    { regex: /(?:password|passwd|pwd)\s*[=:]\s*['"][^'"]{6,}['"]/i, label: 'Hardcoded Password' },
-    { regex: /(?:secret|token|auth[_-]?token|bearer)\s*[=:]\s*['"][A-Za-z0-9_\-\.]{16,}['"]/i, label: 'Secret/Token' },
-    { regex: /ghp_[A-Za-z0-9]{36}/, label: 'GitHub Personal Access Token' },
-    { regex: /sk-[A-Za-z0-9]{20,}/, label: 'OpenAI/Stripe API Key' },
-    { regex: /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/, label: 'JWT Token' },
-    { regex: /(?:SUPABASE|NEXT_PUBLIC|VITE)_\w*_KEY\s*=\s*['"][^'"]{8,}['"]/i, label: 'Framework Secret Key' },
-  ];
-
   // Skip binary/generated files
   const skipExtensions = ['.png', '.jpg', '.gif', '.svg', '.ico', '.woff', '.woff2', '.ttf', '.lock', '.min.js', '.min.css'];
 
@@ -57,41 +152,11 @@ const check = async function scanSecrets(targetDir) {
     const ext = path.extname(file);
     if (skipExtensions.includes(ext)) continue;
 
-    const filePath = path.join(dir, file);
-    if (!fs.existsSync(filePath)) continue;
-
-    let content;
-    try {
-      const buffer = fs.readFileSync(filePath);
-      // Skip binary files (check for null byte in first 512 bytes)
-      let isBinary = false;
-      const checkLen = Math.min(buffer.length, 512);
-      for (let b = 0; b < checkLen; b++) {
-        if (buffer[b] === 0) {
-          isBinary = true;
-          break;
-        }
-      }
-      if (isBinary) continue;
-      content = buffer.toString('utf-8');
-    } catch {
-      continue;
-    }
-
-    const lines = content.split('\n');
-    const fileFindings = [];
-
-    for (let i = 0; i < lines.length; i++) {
-      const line = lines[i];
-      for (const { regex, label } of patterns) {
-        if (regex.test(line)) {
-          const col = line.search(regex);
-          fileFindings.push({ lineNo: i + 1, col, label, line: line.trim().substring(0, 120) });
-          break; // One finding per line
-        }
-      }
-    }
+    // Binary or missing in the snapshot reads as null.
+    const content = readFile(dir, file, mode);
+    if (content === null) continue;
 
+    const fileFindings = findSecrets(content, file);
     if (fileFindings.length > 0) {
       findings[file] = fileFindings;
     }
@@ -111,6 +176,8 @@ const check = async function scanSecrets(targetDir) {
   };
 };
 module.exports = check;
+module.exports.findSecrets = findSecrets;
+module.exports.RULES = RULES;
 
 if (require.main === module) {
   check(process.argv[2]).then(result => {