clckernel 1.2.18 → 2.0.0

package/tools/check_custom.py

@@ -1,233 +1,61 @@
 #!/usr/bin/env python3
 """
 Custom Rules Guard — Python
-Reads .clc-forge.yml and executes user-defined regex rules against changed files.
-Self-contained: uses only Python stdlib + optional pyyaml.
+Reads .clckernel.yaml and executes user-defined regex rules against changed files.
+Self-contained: Python stdlib + optional PyYAML, and the sibling helpers
+_changes.py and _yaml.py (the YAML subset parser, shared with _config.py).
 """
 
+from __future__ import annotations
+
 import json
 import os
 import pathlib
 import re
-import subprocess
 import sys
 from typing import Any, Optional
 
-# ═══════════════════════════════════════════════════════════════════════
-# YAML Parser — try pyyaml first, fallback to inline minimal parser
-# ═══════════════════════════════════════════════════════════════════════
-
-def _parse_yaml(text: str) -> dict:
-    """Parse YAML text. Uses PyYAML if available, otherwise inline minimal parser."""
-    try:
-        import yaml
-        result = yaml.safe_load(text)
-        return result if isinstance(result, dict) else {}
-    except ImportError:
-        pass
-    except Exception:
-        raise
+# Sibling helper shipped next to the guards; works both as a script and when
+# loaded by audit.py through importlib.
+_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+if _TOOLS_DIR not in sys.path:
+    sys.path.insert(0, _TOOLS_DIR)
+import _changes  # noqa: E402
 
-    return _minimal_yaml_parse(text)
 
+def _yaml_module():
+    """The shared parser (tools/_yaml.py), imported only when a config exists.
 
-def _minimal_yaml_parse(text: str) -> dict:
-    """
-    Minimal YAML parser (~80 lines).
-    Supports: scalars, block sequences, block mappings, inline flow sequences,
-    comments. No anchors/aliases, no multiple documents.
+    Install ships it next to _changes.py; an older install without it fails
+    closed (run `clckernel update`).
     """
-    lines = text.split('\n')
-    root: dict = {}
-    # Stack entries: (obj, indent, parent_obj, parent_key)
-    stack: list[tuple[dict, int, Optional[dict], Optional[str]]] = [
-        (root, -1, None, None)
-    ]
-
-    for line_text in lines:
-        # Strip comments (but not inside quoted strings)
-        stripped = line_text.rstrip()
-        comment_idx = _find_comment(stripped)
-        if comment_idx != -1:
-            stripped = stripped[:comment_idx]
-        if not stripped or stripped.isspace():
-            continue
-
-        indent = len(stripped) - len(stripped.lstrip())
-        trimmed = stripped.lstrip()
-
-        # Pop stack back to correct parent
-        while len(stack) > 1 and stack[-1][1] >= indent:
-            stack.pop()
-        entry = stack[-1]
-        obj = entry[0]
-        parent_obj = entry[2]
-        parent_key = entry[3]
-
-        # ── Sequence item: "- rest" ──
-        if trimmed.startswith('- '):
-            rest = trimmed[2:].strip()
-
-            if ': ' in rest:
-                # "- key: value" — mapping inside a sequence
-                arr = _get_or_create_array(parent_obj, parent_key, obj)
-                if arr is not None:
-                    item = {}
-                    colon_idx = rest.index(': ')
-                    k = rest[:colon_idx].strip()
-                    v = rest[colon_idx + 2:].strip()
-                    item[k] = _parse_scalar(v)
-                    arr.append(item)
-                    stack.append((item, indent, arr, None))
-            elif parent_obj is not None and parent_key is not None:
-                # "- scalar" inside a sequence
-                pk = parent_obj.get(parent_key)
-                if isinstance(pk, list):
-                    pk.append(_parse_scalar(rest))
-            continue
-
-        # ── key: value pair ──
-        kv_match = re.match(r'^([^:]+):\s*(.*)$', trimmed)
-        if kv_match:
-            key = kv_match.group(1).strip()
-            val = kv_match.group(2).strip()
-
-            if not val or val == '|' or val == '>':
-                # Empty value → children follow
-                if key not in obj or not isinstance(obj[key], (dict, list)):
-                    obj[key] = {}
-                stack.append((obj[key] if isinstance(obj[key], dict) else {}, indent, obj, key))
-            elif val.startswith('[') and val.endswith(']'):
-                inner = val[1:-1].strip()
-                if inner:
-                    obj[key] = [_parse_scalar(s.strip()) for s in inner.split(',')]
-                else:
-                    obj[key] = []
-            elif val.startswith('{') and val.endswith('}'):
-                obj[key] = _parse_flow_mapping(val)
-            else:
-                obj[key] = _parse_scalar(val)
-            continue
-
-    return root
-
-
-def _find_comment(line: str) -> int:
-    """Find index of comment character not inside quotes."""
-    in_single = False
-    in_double = False
-    for i, ch in enumerate(line):
-        if ch == "'" and not in_double:
-            in_single = not in_single
-        elif ch == '"' and not in_single:
-            in_double = not in_double
-        elif ch == '#' and not in_single and not in_double:
-            return i
-    return -1
-
-
-def _get_or_create_array(parent_obj, parent_key, current_obj):
-    """Get or create an array for sequence items."""
-    if parent_obj is None or parent_key is None:
-        return None
-    pk = parent_obj.get(parent_key)
-    if isinstance(pk, list):
-        return pk
-    if isinstance(pk, dict) and len(pk) == 0:
-        parent_obj[parent_key] = []
-        return parent_obj[parent_key]
-    return None
-
-
-def _parse_scalar(val: str) -> Any:
-    """Parse a YAML scalar value."""
-    if val == '' or val == '~' or val == 'null':
-        return None
-    if val == 'true':
-        return True
-    if val == 'false':
-        return False
-    if re.match(r'^-?\d+$', val):
-        return int(val)
-    if re.match(r'^-?\d+\.\d+$', val):
-        return float(val)
-    # Strip quotes and unescape
-    if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
-        return val[1:-1].replace('\\', '')
-    return val
-
-
-def _parse_flow_mapping(s: str) -> dict:
-    """Parse a YAML inline flow mapping {key: value, ...}."""
-    inner = s[1:-1].strip()
-    if not inner:
-        return {}
-    obj = {}
-    for pair in inner.split(','):
-        colon_idx = pair.find(':')
-        if colon_idx == -1:
-            continue
-        k = pair[:colon_idx].strip()
-        v = pair[colon_idx + 1:].strip()
-        obj[k] = _parse_scalar(v)
-    return obj
+    try:
+        import _yaml
+    except ImportError as exc:
+        raise RuntimeError(
+            "tools/_yaml.py is missing: run `clckernel update` so the guards can read the config"
+        ) from exc
+    return _yaml
 
 
 # ═══════════════════════════════════════════════════════════════════════
-# Glob-to-Regex Converter (~15 lines)
-# Handles: **, *, {a,b} alternation
+# Main Guard Contract
 # ═══════════════════════════════════════════════════════════════════════
 
-import fnmatch
-
-
-def _glob_to_regex(glob_pattern: str) -> re.Pattern:
-    """Convert a glob pattern to a compiled regex. Handles **, *, {a,b}."""
-    # fnmatch.translate converts globs but adds \Z$ at end, we need to adjust
-    # for ** patterns that need to match across path separators
-    parts = glob_pattern.split('/')
-    regex_parts = []
-    i = 0
-    while i < len(parts):
-        part = parts[i]
-        if part == '**':
-            # ** matches zero or more path segments
-            regex_parts.append('(?:.+/)?')
-            # Skip trailing empty segment if glob ends with **
-            i += 1
-            continue
-        else:
-            # Convert single segment with fnmatch
-            translated = fnmatch.translate(part)
-            # fnmatch.translate adds (?s:...) and \Z at end; strip wrapper
-            # Remove leading (?s: and trailing )\Z
-            inner = translated
-            if inner.startswith('(?s:'):
-                inner = inner[4:]
-            if inner.endswith(')\\Z'):
-                inner = inner[:-3]
-            elif inner.endswith('\\Z'):
-                inner = inner[:-2]
-            regex_parts.append(inner)
-        i += 1
-
-    full = '/'.join(regex_parts)
-    return re.compile('^' + full + '$')
-
+# Lookup order: the canonical name first, then older names kept for compatibility.
+CONFIG_NAMES = ['.clckernel.yaml', '.clckernel.yml', '.clc-forge.yaml', '.clc-forge.yml']
+CANONICAL_CONFIG = CONFIG_NAMES[0]
+DEPRECATED_PREFIX = '.clc-forge.'
 
-# ═══════════════════════════════════════════════════════════════════════
-# Main Guard Contract
-# ═══════════════════════════════════════════════════════════════════════
 
 def check(target_dir: Optional[str] = None) -> dict:
     """
     Custom rules guard entry point.
-    Reads .clc-forge.yml and executes user-defined regex rules.
-    Returns: {exit_code, skipped, name, data: {violations, message}}
+    Reads .clckernel.yaml and executes user-defined regex rules.
+    Returns: {exit_code, skipped, name, data: {violations, message[, warnings]}}
     """
     dir_path = pathlib.Path(target_dir or os.getcwd())
-    candidates = ['.clckernel.yml', '.clckernel.yaml', '.clc-forge.yml', '.clc-forge.yaml']
-    config_path = next((dir_path / c for c in candidates if (dir_path / c).exists()), None)
+    config_path = next((dir_path / c for c in CONFIG_NAMES if (dir_path / c).exists()), None)
 
     # ── 1. Load config ──────────────────────────────────────────────
     if not config_path:
@@ -235,27 +63,42 @@ def check(target_dir: Optional[str] = None) -> dict:
             "exit_code": 0,
             "skipped": True,
             "name": "check_custom",
-            "data": {"violations": {}, "message": "No .clckernel.yml or .clc-forge.yml found"},
+            "data": {"violations": {}, "message": f"No {CANONICAL_CONFIG} found"},
+        }
+
+    config_name = config_path.name
+    result = _run_rules(dir_path, config_path, config_name)
+    if config_name.startswith(DEPRECATED_PREFIX):
+        result["data"]["warnings"] = [f"{config_name}: deprecated config name, rename to {CANONICAL_CONFIG}"]
+    return result
+
+
+def _run_rules(dir_path: pathlib.Path, config_path: pathlib.Path, config_name: str) -> dict:
+    """Validate the rules in one config file and run them against changed files."""
+
+    def config_error(message: str) -> dict:
+        return {
+            "exit_code": 1,
+            "skipped": False,
+            "name": "check_custom",
+            "data": {"violations": {}, "message": f"Config error in {config_name}: {message}"},
         }
 
     try:
-        config = _parse_yaml(config_path.read_text(encoding='utf-8'))
+        yaml_module = _yaml_module()
+        config = yaml_module.parse_yaml(config_path.read_text(encoding='utf-8'))
     except Exception as e:
+        # Fail closed: rules we cannot read must not turn into a silent PASS.
         return {
-            "exit_code": 0,
-            "skipped": True,
+            "exit_code": 1,
+            "skipped": False,
             "name": "check_custom",
-            "data": {"violations": {}, "message": f"Config parse error: {e}"},
+            "data": {"violations": {}, "message": f"Config parse error in {config_name}: {e}"},
         }
 
     rules = config.get('rules') if isinstance(config, dict) else None
     if rules is not None and not isinstance(rules, list):
-        return {
-            "exit_code": 0,
-            "skipped": True,
-            "name": "check_custom",
-            "data": {"violations": {}, "message": 'Config error: "rules" must be a list'},
-        }
+        return config_error('"rules" must be a list')
     if not isinstance(rules, list) or len(rules) == 0:
         return {
             "exit_code": 0,
@@ -264,42 +107,40 @@ def check(target_dir: Optional[str] = None) -> dict:
             "data": {"violations": {}, "message": "No custom rules defined"},
         }
 
-    # ── 2. Validate rules: compile regexes, check required fields ───
+    # ── 2. Validate rules: shape and regex; any invalid rule fails the guard ──
     validated = []
-    for rule in rules:
-        if not isinstance(rule, dict):
-            continue
-        if not rule.get('name') or not rule.get('pattern') or not rule.get('message'):
+    errors = []
+    for i, rule in enumerate(rules):
+        problems = _validate_rule(rule)
+        is_named = isinstance(rule, dict) and isinstance(rule.get('name'), str)
+        label = f'"{rule["name"]}"' if is_named else f'#{i + 1}'
+        if problems:
+            errors.append(f"rule {label}: {', '.join(problems)}")
             continue
         try:
             regex = re.compile(rule['pattern'], re.MULTILINE)
-            scope_re = _glob_to_regex(rule.get('scope', '**/*'))
-            exclude_re = [_glob_to_regex(ex) for ex in rule.get('exclude', [])]
-            validated.append({
-                'name': rule['name'],
-                'pattern': rule['pattern'],
-                'message': rule['message'],
-                'severity': rule.get('severity', 'error'),
-                'exclude': rule.get('exclude', []),
-                'regex': regex,
-                'scope_re': scope_re,
-                'exclude_re': exclude_re,
-            })
-        except re.error:
-            # Invalid regex — skip this rule silently
+        except re.error as e:
+            errors.append(f'rule {label}: invalid pattern: {e}')
             continue
-
-    if len(validated) == 0:
-        return {
-            "exit_code": 0,
-            "skipped": False,
-            "name": "check_custom",
-            "data": {"violations": {}, "message": "No valid custom rules after validation"},
-        }
-
-    # ── 3. Get changed files via git diff ───────────────────────────
-    files = _get_changed_files(dir_path)
-    if files is None:
+        validated.append({
+            'name': rule['name'],
+            'pattern': rule['pattern'],
+            'message': rule['message'],
+            'severity': rule.get('severity') or 'error',
+            'exclude': rule.get('exclude') or [],
+            'regex': regex,
+            'scope_re': yaml_module.glob_to_regex(rule.get('scope') or '**/*'),
+            'exclude_re': [yaml_module.glob_to_regex(ex) for ex in rule.get('exclude') or []],
+        })
+
+    if errors:
+        return config_error('; '.join(errors))
+
+    # ── 3. Changed files of the snapshot being verified ─────────────
+    # The index, or HEAD when nothing is staged; never the working copy.
+    changes = _changes.changed_files(str(dir_path))
+    mode, files = changes["mode"], changes["files"]
+    if mode is None:
         return {
             "exit_code": 0,
             "skipped": True,
@@ -319,11 +160,9 @@ def check(target_dir: Optional[str] = None) -> dict:
     has_error = False
 
     for file in files:
-        file_path = pathlib.Path(dir_path) / file
-        try:
-            content = file_path.read_text(encoding='utf-8')
-        except (PermissionError, OSError):
-            continue  # permission denied or unreadable — skip
+        content = _changes.read_file(str(dir_path), file, mode)
+        if content is None:
+            continue  # missing or binary in the snapshot — skip
 
         for rule in validated:
             if not rule['scope_re'].match(file):
@@ -331,6 +170,8 @@ def check(target_dir: Optional[str] = None) -> dict:
             if any(re.match(str(ex.pattern), file) for ex in rule['exclude_re']):
                 continue
 
+            # Empty matches (e.g. `x*`, `^`) are violations too; check_custom.js
+            # steps over them one code point at a time to report the same ones.
             for m in rule['regex'].finditer(content):
                 line_no = content[:m.start()].count('\n') + 1
                 if file not in violations:
@@ -360,28 +201,22 @@ def check(target_dir: Optional[str] = None) -> dict:
     }
 
 
-def _get_changed_files(dir_path: str) -> Optional[list[str]]:
-    """Get list of changed files via git diff. Returns None if no diff available."""
-    try:
-        result = subprocess.run(
-            ['git', 'diff', '--name-only', 'HEAD~1'],
-            cwd=dir_path, capture_output=True, text=True, timeout=10,
-        )
-        files = [f for f in result.stdout.split('\n') if f.strip()]
-        if files:
-            return files
-    except Exception:
-        pass
-
-    try:
-        result = subprocess.run(
-            ['git', 'diff', '--name-only', '--cached'],
-            cwd=dir_path, capture_output=True, text=True, timeout=10,
-        )
-        files = [f for f in result.stdout.split('\n') if f.strip()]
-        return files if files else []
-    except Exception:
-        return None
+def _validate_rule(rule: Any) -> list:
+    """Shape problems of one rule (empty list when valid)."""
+    if not isinstance(rule, dict):
+        return ['must be a mapping']
+    problems = []
+    for field in ('name', 'pattern', 'message'):
+        if not isinstance(rule.get(field), str) or rule.get(field) == '':
+            problems.append(f'"{field}" must be a non-empty string')
+    if rule.get('severity') is not None and rule.get('severity') not in ('error', 'warning'):
+        problems.append('"severity" must be "error" or "warning"')
+    if rule.get('scope') is not None and not isinstance(rule.get('scope'), str):
+        problems.append('"scope" must be a string')
+    exclude = rule.get('exclude')
+    if exclude is not None and not (isinstance(exclude, list) and all(isinstance(e, str) for e in exclude)):
+        problems.append('"exclude" must be a list of strings')
+    return problems
 
 
 # ═══════════════════════════════════════════════════════════════════════
@@ -391,5 +226,7 @@ def _get_changed_files(dir_path: str) -> Optional[list[str]]:
 if __name__ == '__main__':
     target = sys.argv[1] if len(sys.argv) > 1 else None
     result = check(target)
+    for warning in result["data"].get("warnings", []):
+        print(f"[check_custom] warning: {warning}", file=sys.stderr)
     print(json.dumps(result, indent=2))
     sys.exit(result["exit_code"])

package/tools/check_data_files.js

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Data File Guard
+ * Every .json file changed in the snapshot must parse as JSON, whatever folder
+ * it lives in. A failure names the file and the parse error (reason and
+ * line/column); the file content is never printed.
+ * Only JSON: YAML data files may use full YAML, which the strict subset parser
+ * of the harness (tools/yaml.js) cannot validate without false positives.
+ * Self-contained: uses only Node.js built-ins.
+ *
+ * @param {string} targetDir - Repository root (defaults to cwd)
+ * @returns {{ exitCode: number, skipped: boolean, name: string, data: object }}
+ */
+const { changedFiles, readFile } = require('./_changes');
+
+const NAME = 'check_data_files';
+const MAX_REASON = 120;
+
+/** Line and column (1-based) of a character offset. */
+function lineColumn(text, offset) {
+  const before = text.slice(0, offset).split('\n');
+  return { line: before.length, column: before[before.length - 1].length + 1 };
+}
+
+/**
+ * Parse error without the file content: newer V8 messages quote the input
+ * (`..., "<content>" is not valid JSON`), so that part is dropped and the
+ * position, when reported, becomes a line and column.
+ */
+function describeError(err, text) {
+  let reason = String(err && err.message ? err.message : err)
+    .replace(/,\s*"[\s\S]*" is not valid JSON$/, '')
+    .replace(/\s*\(line \d+ column \d+\)$/, '');
+  const position = /\s*(?:in JSON\s*)?at position (\d+)/.exec(reason);
+  let where = '';
+  if (position) {
+    reason = reason.slice(0, position.index);
+    const { line, column } = lineColumn(text, Number(position[1]));
+    where = ` at line ${line}, column ${column}`;
+  }
+  reason = reason.trim().slice(0, MAX_REASON) || 'Invalid JSON';
+  return `Invalid JSON${where}: ${reason}`;
+}
+
+const check = async function checkDataFiles(targetDir) {
+  const dir = targetDir || process.cwd();
+  const { mode, files } = changedFiles(dir, { extensions: ['.json'] });
+
+  if (files.length === 0) {
+    return {
+      exitCode: 0, skipped: true, name: NAME,
+      data: { violations: [], message: 'No changed .json files' },
+    };
+  }
+
+  const violations = [];
+  for (const file of files) {
+    const text = readFile(dir, file, mode);
+    if (text === null) continue;
+    const body = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    try {
+      JSON.parse(body);
+    } catch (err) {
+      violations.push({ file, message: describeError(err, body) });
+    }
+  }
+
+  return {
+    exitCode: violations.length > 0 ? 1 : 0,
+    skipped: false,
+    name: NAME,
+    data: {
+      violations,
+      message: violations.length > 0
+        ? `${violations.length} JSON file(s) do not parse: ${violations.map(v => v.file).join(', ')}`
+        : `${files.length} changed JSON file(s) parse`,
+    },
+  };
+};
+
+module.exports = check;
+
+if (require.main === module) {
+  check(process.argv[2]).then(result => {
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(result.exitCode);
+  });
+}