clckernel 1.2.19 → 2.0.0

package/tools/check_db_efficiency.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 """
-Backend Database Efficiency & N+1 Performance Guard
-AST scanner detecting:
-1. Database queries executed inside loops (N+1 query pattern in SQLAlchemy, Django ORM, SQLModel)
-2. Blocking time.sleep() calls inside async route handlers/coroutines
-3. Unbounded .all() queries missing pagination/limits in API routes
-4. Missing eager loading patterns (select_related, prefetch_related, selectinload, joinedload)
+Backend Database Efficiency & N+1 Heuristic
+Python AST scanner detecting:
+1. ORM-like query calls inside loops/comprehensions (N+1 heuristic for SQLAlchemy, Django ORM, SQLModel)
+2. Blocking time.sleep() calls inside async functions
 
 Self-contained: uses only Python standard library (ast, subprocess, pathlib).
 """
@@ -14,18 +12,61 @@ from __future__ import annotations
 import ast
 import os
 import pathlib
-import subprocess
 import sys
 
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
+
 
 QUERY_METHODS = {
     "filter", "get", "all", "first", "execute", "scalars", "scalar",
     "fetch", "fetchall", "fetchone", "query", "raw"
 }
 
-QUERY_CALLERS = {
-    "db", "session", "objects", "cursor", "connection", "repo", "repository"
-}
+# Receiver names (case-insensitive) that hold an ORM session / connection:
+# session.get(...), db.execute(...), self.session.scalars(...).
+SESSION_NAMES = {"session", "db", "db_session"}
+
+
+def _is_orm_receiver(node: ast.AST) -> bool:
+    """
+    True when a method call on `node` looks like an ORM query:
+      - the attribute chain contains `.objects` (Django manager),
+      - it ends in a session-like name (session, db, db_session; self.session),
+      - it is the result of a `.query(...)` call (SQLAlchemy Query chain),
+      - it is `Model.query` (Flask-SQLAlchemy).
+    Anything else (dict.get, requests.get, config.get) is not a query.
+    """
+    if isinstance(node, ast.Name):
+        return node.id.lower() in SESSION_NAMES
+    if isinstance(node, ast.Attribute):
+        if node.attr == "objects" or node.attr == "query":
+            return True
+        if node.attr.lower() in SESSION_NAMES:
+            # Django's request.session is a dict-like store, not an ORM session.
+            return not (isinstance(node.value, ast.Name) and node.value.id == "request")
+        return _is_orm_receiver(node.value)
+    if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+        if node.func.attr == "query":
+            return True
+        return _is_orm_receiver(node.func.value)
+    return False
+
+
+def _chain_calls(node: ast.AST) -> list[ast.Call]:
+    """Calls inside the receiver chain of `node` (x.query(...).filter(...) -> both calls)."""
+    calls: list[ast.Call] = []
+    while isinstance(node, (ast.Attribute, ast.Call)):
+        if isinstance(node, ast.Call):
+            calls.append(node)
+            node = node.func
+        else:
+            node = node.value
+    return calls
 
 
 class LoopQueryVisitor(ast.NodeVisitor):
@@ -34,6 +75,8 @@ class LoopQueryVisitor(ast.NodeVisitor):
         self.violations: list[dict] = []
         self._loop_depth = 0
         self._is_async_func = False
+        # Calls already covered by an enclosing query chain (one finding per chain).
+        self._reported: set[int] = set()
 
     def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
         prev = self._is_async_func
@@ -84,30 +127,20 @@ class LoopQueryVisitor(ast.NodeVisitor):
                     })
 
         # 2. Detect DB queries inside loops (N+1 query hazard)
-        if self._loop_depth > 0:
-            if isinstance(node.func, ast.Attribute):
-                attr_name = node.func.attr
-                val = node.func.value
-
-                # Match session.execute(), db.query(), Model.objects.filter(), etc.
-                is_db_call = False
-                caller_name = ""
-
-                if isinstance(val, ast.Name):
-                    caller_name = val.id.lower()
-                    if caller_name in QUERY_CALLERS or attr_name in QUERY_METHODS:
-                        is_db_call = True
-                elif isinstance(val, ast.Attribute):
-                    caller_name = val.attr.lower()
-                    if caller_name in QUERY_CALLERS or attr_name in QUERY_METHODS:
-                        is_db_call = True
-
-                if is_db_call and attr_name in QUERY_METHODS:
-                    self.violations.append({
-                        "lineNo": node.lineno,
-                        "rule": "db-query-in-loop",
-                        "message": f"Database query call '.{attr_name}()' detected inside loop/comprehension. Causes N+1 performance bottleneck. Use bulk queries or eager loading (selectinload/joinedload/select_related)."
-                    })
+        if (
+            self._loop_depth > 0
+            and id(node) not in self._reported
+            and isinstance(node.func, ast.Attribute)
+            and node.func.attr in QUERY_METHODS
+            and _is_orm_receiver(node.func.value)
+        ):
+            attr_name = node.func.attr
+            self._reported.update(id(c) for c in _chain_calls(node.func.value))
+            self.violations.append({
+                "lineNo": node.lineno,
+                "rule": "db-query-in-loop",
+                "message": f"Database query call '.{attr_name}()' detected inside loop/comprehension. Causes N+1 performance bottleneck. Use bulk queries or eager loading (selectinload/joinedload/select_related)."
+            })
 
         self.generic_visit(node)
 
@@ -115,32 +148,16 @@ class LoopQueryVisitor(ast.NodeVisitor):
 def check(target_dir: str | None = None) -> dict:
     dir_path = pathlib.Path(target_dir or os.getcwd()).resolve()
 
-    # Get changed python files
-    files: list[str] = []
-    try:
-        out = subprocess.check_output(
-            ["git", "diff", "--name-only", "HEAD~1"],
-            cwd=dir_path,
-            stderr=subprocess.PIPE,
-            text=True
-        )
-        files = [f.strip() for f in out.splitlines() if f.strip().endswith(".py")]
-    except Exception:
-        try:
-            out = subprocess.check_output(
-                ["git", "diff", "--name-only", "--cached"],
-                cwd=dir_path,
-                stderr=subprocess.PIPE,
-                text=True
-            )
-            files = [f.strip() for f in out.splitlines() if f.strip().endswith(".py")]
-        except Exception:
-            return {
-                "exit_code": 0,
-                "skipped": True,
-                "name": "check_db_efficiency",
-                "data": {"message": "No git diff available"}
-            }
+    # The snapshot being verified: the index, or HEAD when nothing is staged.
+    changes = _changes.changed_files(str(dir_path), extensions=[".py"])
+    mode, files = changes["mode"], changes["files"]
+    if mode is None:
+        return {
+            "exit_code": 0,
+            "skipped": True,
+            "name": "check_db_efficiency",
+            "data": {"message": "No git diff available"}
+        }
 
     if not files:
         return {
@@ -153,13 +170,11 @@ def check(target_dir: str | None = None) -> dict:
     violations: dict[str, list[dict]] = {}
 
     for file_rel in files:
-        file_path = dir_path / file_rel
-        if not file_path.is_file():
+        source = _changes.read_file(str(dir_path), file_rel, mode)
+        if source is None:
             continue
-
         try:
-            source = file_path.read_text(encoding="utf-8")
-            tree = ast.parse(source, filename=str(file_path))
+            tree = ast.parse(source, filename=file_rel)
         except Exception:
             continue
 

package/tools/check_hitl.js

@@ -1,97 +1,140 @@
 #!/usr/bin/env node
 /**
  * Human-in-the-Loop (HITL) Checkpoint Guard
- * Ensures subagent playbooks and workflows in skills/ or .agents/skills/
- * define mandatory user review or approval checkpoints before final output.
- * Self-contained: uses only Node.js built-ins.
+ * Every skill and playbook changed in the snapshot must declare its human
+ * checkpoint with an explicit marker:
+ *   - a line starting with `HITL:` (list, quote and bold markup allowed), or
+ *   - a heading `## Human checkpoint` or `## HITL` (any level from 2 to 6).
+ * A word such as "review" or "confirm" is not a marker. Markers inside fenced
+ * code blocks do not count.
+ * Checked files: SKILL.md under skills/, .agents/skills/ or .claude/skills/,
+ * flat skills/*.md files, and Markdown under playbooks/ (README.md excluded).
+ * The kernel's own skill (clckernel_cli) is checked like any other skill, even
+ * though the harness registry lists it (other guards skip registry files).
+ * Ignored: the retired kernel skill (clc_forge_cli, removed by `update`) and
+ * skills vendored by third-party tool plugins (impeccable).
  *
- * @param {string} targetDir - Directory to scan (defaults to cwd)
+ * The project config (.clckernel.yaml) can change all three choices:
+ *   hitl.markers  regexes (case-insensitive, one line, outside code fences)
+ *                 that replace the two markers above
+ *   hitl.paths    globs of the changed files to check, replacing the selection above
+ *   hitl.optOut   { key, value }: a file whose frontmatter has `key: value` is exempt
+ * Self-contained: uses only Node.js built-ins (and the sibling _config.js
+ * when the project has a config file).
+ *
+ * @param {string} targetDir - Repository root (defaults to cwd)
  * @returns {{ exitCode: number, skipped: boolean, name: string, data: object }}
  */
-const check = async function checkHitl(targetDir) {
-  const fs = require('fs');
-  const path = require('path');
+const { changedFiles, readFile, projectConfig, globMatcher } = require('./_changes');
 
-  const dir = targetDir || process.cwd();
-  const searchDirs = [
-    path.join(dir, 'skills'),
-    path.join(dir, '.agents', 'skills')
-  ];
+const NAME = 'check_hitl';
+const SKILL_ROOTS = ['skills/', '.agents/skills/', '.claude/skills/'];
+const EXCLUDED_SKILLS = new Set(['clc_forge_cli', 'impeccable']);
+const FENCE_RE = /^\s{0,3}(`{3,}|~{3,})/;
+const INLINE_MARKER_RE = /^\s*(?:(?:[-*+]|\d+[.)]|>)\s+)*[*_]{0,2}HITL[*_]{0,2}\s*:/i;
+const HEADING_MARKER_RE = /^\s{0,3}#{2,6}\s+(?:Human[- ]checkpoints?|HITL)\b/i;
+
+/** True for a changed file that must carry a HITL marker. */
+function needsMarker(file) {
+  if (!file.endsWith('.md')) return false;
+  const parts = file.split('/');
+  const base = parts[parts.length - 1];
+  if (base.toLowerCase() === 'readme.md') return false;
+  if (parts.some(segment => EXCLUDED_SKILLS.has(segment))) return false;
+  if (file.startsWith('playbooks/')) return true;
+  const root = SKILL_ROOTS.find(r => file.startsWith(r));
+  if (!root) return false;
+  const rest = file.slice(root.length).split('/');
+  // skills/<name>/SKILL.md, or a flat skills/<name>.md file.
+  return base === 'SKILL.md' || rest.length === 1;
+}
+
+const DEFAULT_MARKERS = [INLINE_MARKER_RE, HEADING_MARKER_RE];
 
-  const skillFiles = [];
-  for (const sDir of searchDirs) {
-    if (fs.existsSync(sDir)) {
-      try {
-        const entries = fs.readdirSync(sDir, { withFileTypes: true });
-        for (const entry of entries) {
-          if (entry.isFile() && entry.name.endsWith('.md')) {
-            skillFiles.push(path.join(sDir, entry.name));
-          } else if (entry.isDirectory()) {
-            // Exclude framework CLI scaffolding and tool plugins
-            if (entry.name === 'clckernel_cli' || entry.name === 'clc_forge_cli' || entry.name === 'impeccable') {
-              continue;
-            }
-            const subDirPath = path.join(sDir, entry.name);
-            const subEntries = fs.readdirSync(subDirPath, { withFileTypes: true });
-            for (const sub of subEntries) {
-              if (sub.isFile() && sub.name.endsWith('.md')) {
-                skillFiles.push(path.join(subDirPath, sub.name));
-              }
-            }
-          }
-        }
-      } catch {}
+/** True when the Markdown text holds an explicit HITL marker outside code fences. */
+function hasMarker(text, markers = DEFAULT_MARKERS) {
+  let fence = null;
+  for (const line of text.split(/\r?\n/)) {
+    const open = FENCE_RE.exec(line);
+    if (fence) {
+      if (open && open[1][0] === fence[0] && open[1].length >= fence.length) fence = null;
+      continue;
     }
+    if (open) {
+      fence = open[1];
+      continue;
+    }
+    if (markers.some(re => re.test(line))) return true;
   }
+  return false;
+}
 
-  if (skillFiles.length === 0) {
+/**
+ * Value of a top-level `key:` line of the file's YAML frontmatter, unquoted;
+ * null without frontmatter or key.
+ */
+function frontmatterValue(text, key) {
+  const lines = text.split(/\r?\n/);
+  if (lines[0].trim() !== '---') return null;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i].trim() === '---' || lines[i].trim() === '...') return null;
+    const sep = lines[i].indexOf(':');
+    if (sep <= 0 || lines[i].slice(0, sep) !== key) continue;
+    const value = lines[i].slice(sep + 1).replace(/\s+#.*$/, '').trim();
+    const quoted = /^(['"])(.*)\1$/.exec(value);
+    return quoted ? quoted[2] : value;
+  }
+  return null;
+}
+
+/** The selection, markers and opt-out of this project: config or defaults. */
+function settings(dir) {
+  const config = projectConfig(dir);
+  const hitl = config ? config.hitl : { markers: null, paths: null, optOut: null };
+  return {
+    select: hitl.paths ? globMatcher(hitl.paths) : needsMarker,
+    extensions: hitl.paths ? undefined : ['.md'],
+    markers: hitl.markers ? hitl.markers.map(source => new RegExp(source, 'i')) : DEFAULT_MARKERS,
+    optOut: hitl.optOut,
+  };
+}
+
+const check = async function checkHitl(targetDir) {
+  const dir = targetDir || process.cwd();
+  const { select, extensions, markers, optOut } = settings(dir);
+  const { mode, files } = changedFiles(dir, { extensions, includeManaged: true });
+  const targets = files.filter(select);
+
+  if (targets.length === 0) {
     return {
       exitCode: 0,
       skipped: true,
-      name: 'check_hitl',
-      data: { violations: [], message: 'No skills/ or subagent playbook files found' }
+      name: NAME,
+      data: { violations: [], message: 'No changed skills or playbooks' },
     };
   }
 
-  const hitlKeywords = [
-    /human[- ]in[- ]the[- ]loop/i,
-    /hitl/i,
-    /checkpoint/i,
-    /approval/i,
-    /confirm/i,
-    /review/i,
-    /pausa/i,
-    /aprobaci[oó]n/i,
-    /esperar/i,
-    /validaci[oó]n/i
-  ];
-
   const violations = [];
-  for (const file of skillFiles) {
-    try {
-      const content = fs.readFileSync(file, 'utf-8');
-      const hasHitl = hitlKeywords.some(kw => kw.test(content));
-
-      if (!hasHitl) {
-        violations.push({
-          file: path.relative(dir, file),
-          message: 'Skill/Playbook missing mandatory Human-in-the-Loop (HITL) review checkpoint'
-        });
-      }
-    } catch {}
+  for (const file of targets) {
+    const text = readFile(dir, file, mode);
+    if (text === null || hasMarker(text, markers)) continue;
+    if (optOut && frontmatterValue(text, optOut.key) === optOut.value) continue;
+    violations.push({
+      file,
+      message: 'Missing explicit Human-in-the-Loop checkpoint: add a `HITL:` line or a `## Human checkpoint` section',
+    });
   }
 
-  const exitCode = violations.length > 0 ? 1 : 0;
   return {
-    exitCode,
+    exitCode: violations.length > 0 ? 1 : 0,
     skipped: false,
-    name: 'check_hitl',
+    name: NAME,
     data: {
       violations,
       message: violations.length > 0
-        ? `${violations.length} skill(s) missing mandatory Human-in-the-Loop checkpoints`
-        : 'All subagent skills include mandatory Human-in-the-Loop (HITL) checkpoints'
-    }
+        ? `${violations.length} skill(s)/playbook(s) missing an explicit HITL checkpoint marker`
+        : `All ${targets.length} changed skill(s)/playbook(s) declare a HITL checkpoint`,
+    },
   };
 };
 

package/tools/check_links.js

@@ -0,0 +1,138 @@
+#!/usr/bin/env node
+/**
+ * Relative Link Guard
+ * In every Markdown file changed in the snapshot, relative links
+ * (`[text](path)` and `![alt](path)`) must point to a file or folder present
+ * in the same snapshot (the index on a pre-commit, HEAD on a manual run).
+ * URLs and other schemes (https:, mailto:, ...), pure anchors (#section) and
+ * links inside code spans or fenced code blocks are ignored; the #anchor and
+ * ?query parts of a path are not checked.
+ * Skills the harness installs (clckernel_cli) and registry files are ignored.
+ * Self-contained: uses only Node.js built-ins.
+ *
+ * @param {string} targetDir - Repository root (defaults to cwd)
+ * @returns {{ exitCode: number, skipped: boolean, name: string, data: object }}
+ */
+const path = require('path');
+const { changedFiles, readFile, listFiles } = require('./_changes');
+
+const NAME = 'check_links';
+const HARNESS_SKILLS = new Set(['clckernel_cli', 'clc_forge_cli']);
+const FENCE_RE = /^\s{0,3}(`{3,}|~{3,})/;
+// [text](target "title") / ![alt](<target with spaces>)
+const LINK_RE = /!?\[(?:[^\]\\]|\\.)*\]\(\s*(<[^>\n]*>|[^\s)]+)(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g;
+const SCHEME_RE = /^[a-z][a-z0-9+.-]*:/i;
+
+function isHarnessFile(file) {
+  return file.split('/').some(segment => HARNESS_SKILLS.has(segment));
+}
+
+/**
+ * Markdown lines with fenced code blocks blanked and code spans removed, so
+ * line numbers stay aligned with the file.
+ * @param {string} text
+ * @returns {string[]}
+ */
+function proseLines(text) {
+  let fence = null;
+  return text.split(/\r?\n/).map(line => {
+    const open = FENCE_RE.exec(line);
+    if (fence) {
+      if (open && open[1][0] === fence[0] && open[1].length >= fence.length) fence = null;
+      return '';
+    }
+    if (open) {
+      fence = open[1];
+      return '';
+    }
+    return line.replace(/(`+)[^`]*?\1/g, '');
+  });
+}
+
+/**
+ * Repository path a relative link resolves to, or null when it is not checked.
+ * @returns {string|null} '' for a target outside the repository
+ */
+function resolveTarget(file, raw) {
+  let target = raw.startsWith('<') ? raw.slice(1, -1) : raw;
+  if (!target || target.startsWith('#') || target.startsWith('//') || SCHEME_RE.test(target)) return null;
+  target = target.split('#')[0].split('?')[0];
+  if (!target) return null;
+  try {
+    target = decodeURI(target);
+  } catch {
+    // Keep the raw spelling when it is not valid percent-encoding.
+  }
+  const joined = target.startsWith('/')
+    ? path.posix.normalize(target.slice(1))
+    : path.posix.join(path.posix.dirname(file), target);
+  const normalized = path.posix.normalize(joined).replace(/\/+$/, '');
+  if (normalized === '..' || normalized.startsWith('../')) return '';
+  return normalized === '.' ? '.' : normalized;
+}
+
+const check = async function checkLinks(targetDir) {
+  const dir = targetDir || process.cwd();
+  const { mode, files } = changedFiles(dir, { extensions: ['.md'] });
+  const markdown = files.filter(f => !isHarnessFile(f));
+
+  if (markdown.length === 0) {
+    return {
+      exitCode: 0, skipped: true, name: NAME,
+      data: { violations: [], message: 'No changed Markdown files' },
+    };
+  }
+
+  const snapshot = new Set(listFiles(dir, mode));
+  const folders = new Set(['.']);
+  for (const file of snapshot) {
+    let parent = path.posix.dirname(file);
+    while (parent !== '.' && !folders.has(parent)) {
+      folders.add(parent);
+      parent = path.posix.dirname(parent);
+    }
+  }
+  const present = target => snapshot.has(target) || folders.has(target);
+
+  const violations = [];
+  for (const file of markdown) {
+    const text = readFile(dir, file, mode);
+    if (text === null) continue;
+    proseLines(text).forEach((line, index) => {
+      for (const match of line.matchAll(LINK_RE)) {
+        const target = resolveTarget(file, match[1]);
+        if (target === null) continue;
+        if (target === '' || !present(target)) {
+          violations.push({
+            file,
+            line: index + 1,
+            message: target === ''
+              ? `Link "${match[1]}" points outside the repository`
+              : `Broken link "${match[1]}": ${target} is not in the commit`,
+          });
+        }
+      }
+    });
+  }
+
+  return {
+    exitCode: violations.length > 0 ? 1 : 0,
+    skipped: false,
+    name: NAME,
+    data: {
+      violations,
+      message: violations.length > 0
+        ? `${violations.length} broken relative link(s) found`
+        : `All relative links in ${markdown.length} changed Markdown file(s) resolve`,
+    },
+  };
+};
+
+module.exports = check;
+
+if (require.main === module) {
+  check(process.argv[2]).then(result => {
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(result.exitCode);
+  });
+}