clckernel 1.2.19 → 2.0.1

package/tools/_changes.py

@@ -0,0 +1,233 @@
+"""
+Snapshot helpers shared by the diff-based guards (twin of _changes.js).
+
+A guard evaluates exactly the snapshot being verified:
+  - "staged":      something is staged (a pre-commit) -> the index
+  - "last-commit": nothing staged (a manual run)      -> HEAD, diffed against HEAD~1
+                   (a root commit lists every file of HEAD)
+  - None:          no git repository / git unavailable
+File content always comes from that snapshot, never from the working copy.
+
+Files the harness installed itself are listed in tools/.clckernel-managed.json
+(written by `clckernel` install/update). changed_files drops every listed path,
+so no guard evaluates the harness's own code. Without a registry (e.g. the
+clckernel repository, where tools/ is source code) nothing is excluded.
+
+The project's `exclude_paths` (.clckernel.yaml) are dropped too, for every
+guard. The config is read through the sibling _config.py, which install ships
+next to this file; an older install without it fails closed when a config file
+appears (run `clckernel update`). An invalid config raises its
+ConfigError: a guard never runs on a config it cannot read.
+
+Self-contained: Python 3.9+ standard library only. Git runs with argument
+lists, never through a shell; paths are read NUL-separated (-z), so spaces and
+non-ASCII names survive.
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from typing import Callable, Optional
+
+BINARY_PROBE_BYTES = 8000
+MANAGED_REGISTRY = "tools/.clckernel-managed.json"
+CONFIG_NAMES = ('.clckernel.yaml', '.clckernel.yml', '.clc-forge.yaml', '.clc-forge.yml')
+
+
+def project_config(target_dir: Optional[str] = None) -> Optional[dict]:
+    """
+    The project's validated config (defaults filled in), or None when the
+    project has no config file. Deprecation warnings go to stderr once.
+
+    Raises ConfigError on an invalid config, RuntimeError when the sibling
+    _config.py (or _yaml.py) is missing.
+    """
+    root = str(target_dir or os.getcwd())
+    if not any(os.path.exists(os.path.join(root, name)) for name in CONFIG_NAMES):
+        return None
+    tools_dir = os.path.dirname(os.path.abspath(__file__))
+    if tools_dir not in sys.path:
+        sys.path.insert(0, tools_dir)
+    try:
+        import _config
+    except ImportError as exc:
+        raise RuntimeError(
+            f"The project config needs tools/_config.py and tools/_yaml.py ({exc}): "
+            "run `clckernel update` so the guards can read it"
+        ) from exc
+    loaded = _config.load_config(root)
+    _config.report_warnings(loaded)
+    return loaded["config"]
+
+
+def glob_matcher(globs) -> Callable[[str], bool]:
+    """A predicate for root-relative paths matching one of the globs (always False without globs)."""
+    if not globs:
+        return lambda rel: False
+    import _config
+    return _config.glob_matcher(globs)
+
+
+def _git(cwd: str, *args: str) -> Optional[bytes]:
+    """Run git in cwd; stdout bytes on success, None on any failure."""
+    try:
+        proc = subprocess.run(
+            ["git", *args], cwd=cwd,
+            stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+        )
+    except (OSError, ValueError):
+        return None
+    return proc.stdout if proc.returncode == 0 else None
+
+
+def _split_nul(out: bytes) -> list:
+    return [p for p in out.decode("utf-8", errors="surrogateescape").split("\0") if p]
+
+
+def _has_commit(cwd: str, rev: str) -> bool:
+    return _git(cwd, "rev-parse", "--verify", "--quiet", f"{rev}^{{commit}}") is not None
+
+
+def changed_files(target_dir: Optional[str] = None, extensions=None) -> dict:
+    """
+    List the files added, copied, modified or renamed in the snapshot.
+
+    Returns {"mode": "staged" | "last-commit" | None, "files": [...]}, minus
+    the registry-listed files and the config's exclude_paths; when extensions
+    is given, only paths ending in one of them are kept.
+    """
+    cwd = str(target_dir or os.getcwd())
+    any_staged = _git(cwd, "diff", "--cached", "--name-only", "-z")
+    if any_staged is None:
+        return {"mode": None, "files": []}
+
+    if any_staged:
+        mode = "staged"
+        out = _git(cwd, "diff", "--cached", "--name-only", "-z", "--diff-filter=ACMR")
+    elif not _has_commit(cwd, "HEAD"):
+        # No commits and nothing staged: the index holds nothing to evaluate.
+        return {"mode": "staged", "files": []}
+    else:
+        mode = "last-commit"
+        if _has_commit(cwd, "HEAD~1"):
+            out = _git(cwd, "diff", "--name-only", "-z", "--diff-filter=ACMR", "HEAD~1", "HEAD")
+        else:
+            out = _git(cwd, "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD")
+    if out is None:
+        return {"mode": None, "files": []}
+
+    managed = _managed_files(cwd, mode)
+    config = project_config(cwd)
+    excluded = glob_matcher(config["exclude_paths"] if config else [])
+    files = [f for f in _split_nul(out) if f not in managed and not excluded(f)]
+    if extensions:
+        files = [f for f in files if f.endswith(tuple(extensions))]
+    return {"mode": mode, "files": files}
+
+
+def pure_renames(target_dir: Optional[str], mode: Optional[str]) -> set:
+    """
+    New paths of the PURE renames in the snapshot: files whose content is
+    identical to a file removed in the same change (same blob, git R100). The
+    pair compared is the one changed_files uses: HEAD -> index when staged,
+    HEAD~1 -> HEAD in last-commit mode; a root commit (or an index with no HEAD)
+    has no renames. A rename with any content change is not listed.
+    """
+    cwd = str(target_dir or os.getcwd())
+    raw = ("--raw", "-z", "--no-abbrev", "-M", "--diff-filter=R")
+    out = None
+    if mode == "staged" and _has_commit(cwd, "HEAD"):
+        out = _git(cwd, "diff", "--cached", *raw, "HEAD")
+    elif mode == "last-commit" and _has_commit(cwd, "HEAD~1"):
+        out = _git(cwd, "diff", *raw, "HEAD~1", "HEAD")
+    renamed: set = set()
+    if out is None:
+        return renamed
+    # Each record: ":<src mode> <dst mode> <src blob> <dst blob> R<score>" NUL old NUL new NUL.
+    parts = _split_nul(out)
+    for i in range(0, len(parts) - 2, 3):
+        fields = parts[i].lstrip(":").split(" ")
+        if len(fields) == 5 and fields[2] == fields[3]:
+            renamed.add(parts[i + 2])
+    return renamed
+
+
+def _managed_files(cwd: str, mode: Optional[str]) -> set:
+    """
+    Paths listed in the harness registry: read from the snapshot being evaluated,
+    falling back to the working copy at the repository root. A missing or
+    malformed registry excludes nothing.
+    """
+    text = read_file(cwd, MANAGED_REGISTRY, mode)
+    if text is None and mode is not None:
+        top = _git(cwd, "rev-parse", "--show-toplevel")
+        root = top.decode("utf-8", errors="surrogateescape").strip() if top else cwd
+        text = read_file(root, MANAGED_REGISTRY, None)
+    if text is None:
+        return set()
+    try:
+        data = json.loads(text)
+    except ValueError:
+        return set()
+    # Older installs wrote a list of paths; current ones {kernelVersion, files: [{path, sha256}]}.
+    if isinstance(data, list):
+        return {f for f in data if isinstance(f, str)}
+    if isinstance(data, dict) and isinstance(data.get("files"), list):
+        paths = {f.get("path") for f in data["files"] if isinstance(f, dict)}
+        return {p for p in paths if isinstance(p, str)} | {MANAGED_REGISTRY}
+    return set()
+
+
+def _object_name(file: str, mode: str) -> str:
+    """Object name of a path in the snapshot (paths are relative to the repo root)."""
+    return f":0:{file}" if mode == "staged" else f"HEAD:{file}"
+
+
+def read_file(target_dir: Optional[str], file: str, mode: Optional[str]) -> Optional[str]:
+    """
+    Read a file from the snapshot: UTF-8 text, or None when missing or binary.
+    With mode None (no git) the working copy is the only source there is.
+    """
+    cwd = str(target_dir or os.getcwd())
+    if mode is None:
+        try:
+            with open(os.path.join(cwd, file), "rb") as fh:
+                data = fh.read()
+        except OSError:
+            return None
+    else:
+        data = _git(cwd, "cat-file", "blob", _object_name(file, mode))
+        if data is None:
+            return None
+    if b"\0" in data[:BINARY_PROBE_BYTES]:
+        return None
+    return data.decode("utf-8", errors="replace")
+
+
+def exists(target_dir: Optional[str], file: str, mode: Optional[str]) -> bool:
+    """True when the path exists in the snapshot (the working copy when mode is None)."""
+    cwd = str(target_dir or os.getcwd())
+    if mode is None:
+        return os.path.exists(os.path.join(cwd, file))
+    return _git(cwd, "cat-file", "-e", _object_name(file, mode)) is not None
+
+
+def list_files(target_dir: Optional[str], mode: Optional[str]) -> list:
+    """
+    List every file of the snapshot, for whole-repository facts (e.g. a name
+    that must be unique, a link target that must exist): the index in staged
+    mode, HEAD in last-commit mode. Paths are relative to the repository root
+    and sorted; nothing is excluded. With mode None (no git) nothing is listed.
+    """
+    cwd = str(target_dir or os.getcwd())
+    if mode == "staged":
+        out = _git(cwd, "ls-files", "-z", "--cached", "--full-name", "--", ":/")
+    elif mode == "last-commit":
+        out = _git(cwd, "ls-tree", "-r", "-z", "--name-only", "--full-tree", "HEAD")
+    else:
+        return []
+    if out is None:
+        return []
+    return sorted(set(_split_nul(out)))

package/tools/_config.js

@@ -0,0 +1,294 @@
+/**
+ * Project configuration shared by install/update and the guards (twin of
+ * _config.py): finds .clckernel.yaml in the project root, parses it with
+ * tools/yaml.js, validates it and fills in the defaults. The defaults
+ * reproduce the behaviour of a project without the key.
+ *
+ * Schema (every key optional):
+ *   rules             custom regex rules, run by check_custom
+ *   exclude_paths     globs no diff-based guard evaluates (_changes)
+ *   mirrors.list      AI IDE mirrors install creates and check_symlinks verifies
+ *                     (null = each one's built-in list)
+ *   mirrors.required  true: a listed mirror that is missing fails check_symlinks
+ *   hitl.markers      regexes (case-insensitive) that count as a HITL marker
+ *                     (null = the built-in markers)
+ *   hitl.paths        globs of the files check_hitl evaluates (null = built-in)
+ *   hitl.optOut       { key, value }: a file whose frontmatter has key: value is exempt
+ *   update.exclude    globs install/update never write
+ *   tdd.code          extra globs verify_tdd counts as production code
+ *   tdd.ignore        globs verify_tdd never counts as production code
+ *   severities        { <guard name>: warning | error }: the level of a failing
+ *                     guard in the audit orchestrators (warning = reported,
+ *                     not blocking; a crash is always an error)
+ *
+ * Removed keys (no effect) produce a deprecation warning; unknown top-level
+ * keys a warning; an invalid value throws ConfigError naming file and key.
+ * Self-contained: Node.js built-ins plus the sibling yaml.js.
+ */
+const fs = require('fs');
+const path = require('path');
+const { parseYaml, globToRegex } = require('./yaml.js');
+
+// Lookup order: the canonical name first, then older names kept for compatibility.
+const CONFIG_NAMES = ['.clckernel.yaml', '.clckernel.yml', '.clc-forge.yaml', '.clc-forge.yml'];
+
+const KEYS = ['rules', 'exclude_paths', 'mirrors', 'hitl', 'update', 'tdd', 'severities'];
+const LEVELS = ['error', 'warning'];
+
+/** Keys earlier releases documented that never had an effect, with the reason. */
+const DEPRECATED_KEYS = {
+  active_guards: 'guards are chosen by the stack, the registry and `clckernel update --add-guards`',
+  phases: 'the workflow phases are agent instructions in AGENTS.md',
+  gate_mode: 'every installed guard blocks the commit when it fails',
+  layers: 'layer rules live in the architecture guards',
+  scope: 'use exclude_paths to keep paths out of the guards',
+};
+
+class ConfigError extends Error {
+  /**
+   * @param {string} key - Dotted key path, e.g. "mirrors.list[0]" ("<root>" for the document)
+   * @param {string} message - What is wrong
+   * @param {string} [file] - Config file name
+   * @param {string[]} [allowed] - Accepted values, when a closed set
+   */
+  constructor(key, message, file = CONFIG_NAMES[0], allowed = []) {
+    super(`Invalid ${file}: key "${key}" — ${message}` + (allowed.length ? ` (allowed: ${allowed.join(', ')})` : ''));
+    this.name = 'ConfigError';
+    this.key = key;
+    this.path = key;
+    this.file = file;
+    this.allowed = allowed;
+    this.fix = allowed.length
+      ? `Set "${key}" in ${file} to one of: ${allowed.join(', ')}.`
+      : `Correct "${key}" in ${file}: ${message}.`;
+  }
+}
+
+function defaultConfig() {
+  return {
+    rules: [],
+    exclude_paths: [],
+    mirrors: { list: null, required: false },
+    hitl: { markers: null, paths: null, optOut: null },
+    update: { exclude: [] },
+    tdd: { code: [], ignore: [] },
+    severities: {},
+  };
+}
+
+const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
+const typeName = (v) => (v === null ? 'null' : Array.isArray(v) ? 'a list' : typeof v);
+
+function stringList(value, key, file, check) {
+  if (!Array.isArray(value)) throw new ConfigError(key, `expected a list of strings, got ${typeName(value)}`, file);
+  value.forEach((item, i) => {
+    if (typeof item !== 'string' || item.trim() === '') {
+      throw new ConfigError(`${key}[${i}]`, 'each item must be a non-empty string', file);
+    }
+    if (check) check(item, `${key}[${i}]`);
+  });
+  return [...value];
+}
+
+/** Reads a nested mapping key, rejecting sub-keys the schema does not know. */
+function section(raw, key, known, file, label = key) {
+  const value = raw[key];
+  if (value === undefined || value === null) return {};
+  if (!isMapping(value)) throw new ConfigError(label, `expected a mapping, got ${typeName(value)}`, file);
+  for (const sub of Object.keys(value)) {
+    if (!known.includes(sub)) throw new ConfigError(`${label}.${sub}`, 'unknown key', file, known.map((k) => `${label}.${k}`));
+  }
+  return value;
+}
+
+function validateRules(rules, file) {
+  if (rules === undefined || rules === null) return [];
+  if (!Array.isArray(rules)) throw new ConfigError('rules', `expected a list, got ${typeName(rules)}`, file);
+  const seen = new Set();
+  rules.forEach((rule, i) => {
+    if (!isMapping(rule)) throw new ConfigError(`rules[${i}]`, 'each rule must be a mapping', file);
+    if (typeof rule.name !== 'string' || rule.name.trim() === '') {
+      throw new ConfigError(`rules[${i}].name`, 'each rule requires a non-empty name', file);
+    }
+    if (seen.has(rule.name)) throw new ConfigError(`rules[${i}].name`, `duplicate rule name "${rule.name}"`, file);
+    seen.add(rule.name);
+    if (rule.severity !== undefined && rule.severity !== null && !['error', 'warning'].includes(rule.severity)) {
+      throw new ConfigError(`rules[${i}].severity`, `invalid severity "${rule.severity}"`, file, ['error', 'warning']);
+    }
+  });
+  return rules;
+}
+
+/** severities: a mapping of guard name -> 'error' | 'warning'. */
+function validateSeverities(value, file) {
+  if (value === undefined || value === null) return {};
+  if (!isMapping(value)) throw new ConfigError('severities', `expected a mapping of guard name to level, got ${typeName(value)}`, file);
+  const out = {};
+  for (const [name, level] of Object.entries(value)) {
+    if (name.trim() === '') throw new ConfigError('severities', 'guard names must be non-empty', file);
+    if (!LEVELS.includes(level)) throw new ConfigError(`severities.${name}`, `invalid level ${JSON.stringify(level)}`, file, LEVELS);
+    out[name] = level;
+  }
+  return out;
+}
+
+function checkMirrorPath(item, key, file) {
+  const segments = item.split(/[\\/]/);
+  if (path.isAbsolute(item) || item.startsWith('/') || segments.includes('..')) {
+    throw new ConfigError(key, `"${item}" must be a path inside the project`, file);
+  }
+  if (path.posix.normalize(item) === 'AGENTS.md') throw new ConfigError(key, 'AGENTS.md is the source, not a mirror', file);
+}
+
+function checkRegex(item, key, file) {
+  try {
+    new RegExp(item, 'i');
+  } catch (err) {
+    throw new ConfigError(key, `invalid regular expression: ${err.message}`, file);
+  }
+}
+
+/**
+ * Validates a parsed config and fills in the defaults.
+ * @param {object} raw - Parsed YAML (top-level mapping)
+ * @param {string} [file] - Config file name, for messages
+ * @returns {{ config: object, warnings: string[] }}
+ * @throws {ConfigError}
+ */
+function validateConfig(raw, file = CONFIG_NAMES[0]) {
+  if (!isMapping(raw)) throw new ConfigError('<root>', 'expected a mapping at the top level', file);
+  const config = defaultConfig();
+  const warnings = [];
+  for (const key of Object.keys(raw)) {
+    if (Object.prototype.hasOwnProperty.call(DEPRECATED_KEYS, key)) {
+      warnings.push(`${file}: key "${key}" is deprecated and has no effect (${DEPRECATED_KEYS[key]}); remove it`);
+    } else if (!KEYS.includes(key)) {
+      warnings.push(`${file}: unknown key "${key}" is ignored (known keys: ${KEYS.join(', ')})`);
+    }
+  }
+
+  config.rules = validateRules(raw.rules, file);
+  if (raw.exclude_paths !== undefined && raw.exclude_paths !== null) {
+    config.exclude_paths = stringList(raw.exclude_paths, 'exclude_paths', file);
+  }
+
+  const mirrors = section(raw, 'mirrors', ['list', 'required'], file);
+  if (mirrors.list !== undefined && mirrors.list !== null) {
+    config.mirrors.list = stringList(mirrors.list, 'mirrors.list', file, (item, key) => checkMirrorPath(item, key, file));
+  }
+  if (mirrors.required !== undefined && mirrors.required !== null) {
+    if (typeof mirrors.required !== 'boolean') throw new ConfigError('mirrors.required', `expected true or false, got ${typeName(mirrors.required)}`, file);
+    config.mirrors.required = mirrors.required;
+  }
+
+  const hitl = section(raw, 'hitl', ['markers', 'paths', 'optOut'], file);
+  if (hitl.markers !== undefined && hitl.markers !== null) {
+    config.hitl.markers = stringList(hitl.markers, 'hitl.markers', file, (item, key) => checkRegex(item, key, file));
+  }
+  if (hitl.paths !== undefined && hitl.paths !== null) config.hitl.paths = stringList(hitl.paths, 'hitl.paths', file);
+  if (hitl.optOut !== undefined && hitl.optOut !== null) {
+    const optOut = section(hitl, 'optOut', ['key', 'value'], file, 'hitl.optOut');
+    if (typeof optOut.key !== 'string' || optOut.key.trim() === '') {
+      throw new ConfigError('hitl.optOut.key', 'expected a non-empty frontmatter key', file);
+    }
+    if (!['string', 'number', 'boolean'].includes(typeof optOut.value)) {
+      throw new ConfigError('hitl.optOut.value', 'expected the frontmatter value that exempts a file', file);
+    }
+    config.hitl.optOut = { key: optOut.key, value: String(optOut.value) };
+  }
+
+  const update = section(raw, 'update', ['exclude'], file);
+  if (update.exclude !== undefined && update.exclude !== null) config.update.exclude = stringList(update.exclude, 'update.exclude', file);
+
+  const tdd = section(raw, 'tdd', ['code', 'ignore'], file);
+  if (tdd.code !== undefined && tdd.code !== null) config.tdd.code = stringList(tdd.code, 'tdd.code', file);
+  if (tdd.ignore !== undefined && tdd.ignore !== null) config.tdd.ignore = stringList(tdd.ignore, 'tdd.ignore', file);
+
+  config.severities = validateSeverities(raw.severities, file);
+
+  return { config, warnings };
+}
+
+/**
+ * Parses config text.
+ * @returns {object} The top-level mapping ({} for an empty document)
+ * @throws {ConfigError}
+ */
+function parseConfigText(text, file = CONFIG_NAMES[0]) {
+  if (text === undefined || text === null || String(text).trim() === '') return {};
+  let raw;
+  try {
+    raw = parseYaml(String(text));
+  } catch (err) {
+    throw new ConfigError('<root>', `YAML parse error: ${err.message}`, file);
+  }
+  if (raw === null || raw === undefined) return {};
+  if (!isMapping(raw)) throw new ConfigError('<root>', 'expected a mapping at the top level', file);
+  return raw;
+}
+
+/** Absolute path of the project's config file, or null. */
+function findConfigFile(dir) {
+  const root = dir || process.cwd();
+  const found = CONFIG_NAMES.map((name) => path.join(root, name)).find((p) => fs.existsSync(p));
+  return found || null;
+}
+
+const cache = new Map();
+
+/**
+ * Loads the config of the project at dir (read from the working tree).
+ * @param {string} [dir] - Project root (default: cwd)
+ * @param {{ cache?: boolean }} [options] - Reuse the result for dir within this process
+ * @returns {{ file: string|null, path: string|null, config: object, warnings: string[] }}
+ * @throws {ConfigError}
+ */
+function loadConfig(dir, { cache: useCache = true } = {}) {
+  const root = path.resolve(dir || process.cwd());
+  if (useCache && cache.has(root)) return cache.get(root);
+  const configPath = findConfigFile(root);
+  let loaded;
+  if (!configPath) {
+    loaded = { file: null, path: null, config: defaultConfig(), warnings: [] };
+  } else {
+    const file = path.basename(configPath);
+    const { config, warnings } = validateConfig(parseConfigText(fs.readFileSync(configPath, 'utf-8'), file), file);
+    loaded = { file, path: configPath, config, warnings };
+  }
+  if (useCache) cache.set(root, loaded);
+  return loaded;
+}
+
+/**
+ * A predicate telling whether a root-relative POSIX path matches one of the globs.
+ * @param {string[]} globs
+ * @returns {(rel: string) => boolean}
+ */
+function globMatcher(globs) {
+  const regexes = (globs || []).map(globToRegex);
+  return (rel) => regexes.some((re) => re.test(rel));
+}
+
+const reported = new Set();
+
+/** Prints each warning once per process, on stderr (guards keep stdout for JSON). */
+function reportWarnings(loaded) {
+  for (const warning of loaded.warnings) {
+    if (reported.has(warning)) continue;
+    reported.add(warning);
+    process.stderr.write(`clckernel: warning: ${warning}\n`);
+  }
+}
+
+module.exports = {
+  CONFIG_NAMES,
+  DEPRECATED_KEYS,
+  ConfigError,
+  defaultConfig,
+  validateConfig,
+  parseConfigText,
+  findConfigFile,
+  loadConfig,
+  globMatcher,
+  reportWarnings,
+};