@deftai/directive-content 0.55.2 → 0.56.0

package/scripts/_cache_validate.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""_cache_validate.py -- in-module mirror of cache-meta.schema.json (#883 Story 2).
+
+Hand-rolled validator (no third-party ``jsonschema`` dependency) that pins
+the same field set / types / enums declared in
+``vbrief/schemas/cache-meta.schema.json``. Drift between the schema file
+and this validator is caught by the
+``tests/content/test_cache_schema_alignment.py`` regression suite.
+
+Extracted from :mod:`cache` to keep the parent module under the 1000-line
+MUST limit from deft/main.md (the cache module is a multi-command CLI plus
+fetch orchestrator; the schema validator is a self-contained sub-surface).
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from typing import Any
+
+
+class CacheValidationError(ValueError):
+    """meta.json failed schema validation on read or write."""
+
+
+# ---------------------------------------------------------------------------
+# Regex + frozen-set surface
+# ---------------------------------------------------------------------------
+
+_VALID_DATETIME_RE: re.Pattern[str] = re.compile(
+    r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(Z|[+-]\d{2}:\d{2})$"
+)
+_VALID_SEMVER_RE: re.Pattern[str] = re.compile(r"^\d+\.\d+\.\d+$")
+
+_META_REQUIRED: tuple[str, ...] = (
+    "source",
+    "key",
+    "fetched_at",
+    "ttl_seconds",
+    "expires_at",
+    "scan_result",
+    "size_bytes",
+    "stale",
+)
+_META_ALLOWED: frozenset[str] = frozenset(_META_REQUIRED) | {"etag"}
+_SCAN_RESULT_REQUIRED: tuple[str, ...] = (
+    "passed",
+    "scanned_at",
+    "scanner_version",
+    "flags",
+)
+_SCAN_RESULT_ALLOWED: frozenset[str] = frozenset(_SCAN_RESULT_REQUIRED)
+_SCAN_FLAG_REQUIRED: tuple[str, ...] = ("category", "severity", "detail")
+_SCAN_FLAG_ALLOWED: frozenset[str] = frozenset(_SCAN_FLAG_REQUIRED) | {"match_count"}
+_SCAN_FLAG_CATEGORIES: frozenset[str] = frozenset(
+    {"injection-heading", "credentials", "invisible-unicode"}
+)
+_SCAN_FLAG_SEVERITIES: frozenset[str] = frozenset(
+    {"fence-and-pass", "hard-fail", "strip-and-pass"}
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _require_keys(obj: dict[str, Any], required: Iterable[str], path: str) -> None:
+    missing = [k for k in required if k not in obj]
+    if missing:
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}: missing required keys {missing!r}"
+        )
+
+
+def _disallow_extras(obj: dict[str, Any], allowed: frozenset[str], path: str) -> None:
+    extra = set(obj) - allowed
+    if extra:
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}: unknown keys {sorted(extra)!r}"
+        )
+
+
+def _is_non_negative_int(value: object) -> bool:
+    """Reject bools (which are int subclass in Python) and negative ints."""
+    return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+
+def _validate_datetime(value: object, path: str) -> None:
+    if not isinstance(value, str) or not _VALID_DATETIME_RE.match(value):
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}: not a UTC-suffixed "
+            f"ISO-8601 timestamp ({value!r})"
+        )
+
+
+def _validate_meta_envelope(meta: dict[str, Any], allowed_sources: tuple[str, ...]) -> None:
+    """Top-level (source/key/timestamps/ttl/size/stale) checks. Raises on failure."""
+    if meta["source"] not in allowed_sources:
+        raise CacheValidationError(
+            f"meta.json validation failure at .source: {meta['source']!r} not in "
+            f"{sorted(allowed_sources)!r}"
+        )
+    if not isinstance(meta["key"], str) or not meta["key"]:
+        raise CacheValidationError(
+            "meta.json validation failure at .key: expected non-empty string"
+        )
+    _validate_datetime(meta["fetched_at"], ".fetched_at")
+    _validate_datetime(meta["expires_at"], ".expires_at")
+    if not _is_non_negative_int(meta["ttl_seconds"]):
+        raise CacheValidationError(
+            f"meta.json validation failure at .ttl_seconds: expected non-negative int "
+            f"(got {meta['ttl_seconds']!r})"
+        )
+    if not _is_non_negative_int(meta["size_bytes"]):
+        raise CacheValidationError(
+            f"meta.json validation failure at .size_bytes: expected non-negative int "
+            f"(got {meta['size_bytes']!r})"
+        )
+    if not isinstance(meta["stale"], bool):
+        raise CacheValidationError(
+            f"meta.json validation failure at .stale: expected bool (got {meta['stale']!r})"
+        )
+    if "etag" in meta and not isinstance(meta["etag"], str):
+        raise CacheValidationError(
+            f"meta.json validation failure at .etag: expected string when present "
+            f"(got {meta['etag']!r})"
+        )
+
+
+def _validate_scan_result(scan_result: object) -> None:
+    if not isinstance(scan_result, dict):
+        raise CacheValidationError(
+            "meta.json validation failure at .scan_result: expected object"
+        )
+    _require_keys(scan_result, _SCAN_RESULT_REQUIRED, ".scan_result")
+    _disallow_extras(scan_result, _SCAN_RESULT_ALLOWED, ".scan_result")
+    if not isinstance(scan_result["passed"], bool):
+        raise CacheValidationError(
+            "meta.json validation failure at .scan_result.passed: expected bool"
+        )
+    _validate_datetime(scan_result["scanned_at"], ".scan_result.scanned_at")
+    sv = scan_result["scanner_version"]
+    if not isinstance(sv, str) or not _VALID_SEMVER_RE.match(sv):
+        raise CacheValidationError(
+            "meta.json validation failure at .scan_result.scanner_version: not a "
+            f"SemVer string ({sv!r})"
+        )
+    flags = scan_result["flags"]
+    if not isinstance(flags, list):
+        raise CacheValidationError(
+            "meta.json validation failure at .scan_result.flags: expected array"
+        )
+    for i, flag in enumerate(flags):
+        _validate_scan_flag(flag, i)
+
+
+def _validate_scan_flag(flag: object, index: int) -> None:
+    path = f".scan_result.flags[{index}]"
+    if not isinstance(flag, dict):
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}: expected object"
+        )
+    _require_keys(flag, _SCAN_FLAG_REQUIRED, path)
+    _disallow_extras(flag, _SCAN_FLAG_ALLOWED, path)
+    if flag["category"] not in _SCAN_FLAG_CATEGORIES:
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}.category: "
+            f"{flag['category']!r} not in {sorted(_SCAN_FLAG_CATEGORIES)!r}"
+        )
+    if flag["severity"] not in _SCAN_FLAG_SEVERITIES:
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}.severity: "
+            f"{flag['severity']!r} not in {sorted(_SCAN_FLAG_SEVERITIES)!r}"
+        )
+    if not isinstance(flag["detail"], str):
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}.detail: expected string"
+        )
+    if "match_count" in flag and not _is_non_negative_int(flag["match_count"]):
+        raise CacheValidationError(
+            f"meta.json validation failure at {path}.match_count: expected "
+            f"non-negative int (got {flag['match_count']!r})"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+def validate_meta(meta: dict[str, Any], allowed_sources: tuple[str, ...]) -> None:
+    """Validate ``meta`` against cache-meta.schema.json. Raises :class:`CacheValidationError`.
+
+    Args:
+        meta: The parsed meta.json object.
+        allowed_sources: The cache layer's :data:`cache.ALLOWED_SOURCES` tuple. Threaded
+            through as a parameter (rather than imported) so this module
+            is independent of the parent cache module's import graph.
+    """
+    if not isinstance(meta, dict):
+        raise CacheValidationError(
+            f"meta.json validation failure at <root>: expected object, got "
+            f"{type(meta).__name__}"
+        )
+    _require_keys(meta, _META_REQUIRED, "<root>")
+    _disallow_extras(meta, _META_ALLOWED, "<root>")
+    _validate_meta_envelope(meta, allowed_sources)
+    _validate_scan_result(meta["scan_result"])

package/scripts/_content_root.py

@@ -0,0 +1,42 @@
+"""_content_root.py -- resolve the shippable-content root across both contexts (#1875).
+
+The #1875 "content/ move" relocated every shippable framework asset under a
+single ``content/`` root in the SOURCE repository. The C1 flatten deposit
+(``build_dist.py``) strips that prefix when packaging, so a CONSUMER install
+sees the same ``.deft/core/<x>`` layout it always has -- there is no
+``content/`` directory in a deposited framework.
+
+Engine scripts that read shippable content by repo-root path therefore live in
+two worlds:
+
+* SOURCE checkout: content lives at ``<framework-root>/content/<x>``.
+* CONSUMER deposit: content lives at ``<framework-root>/<x>`` (flattened).
+
+``content_root(framework_root)`` resolves the difference by probing for the
+``content/`` directory: it returns ``<framework-root>/content`` when that
+directory exists (source) and ``<framework-root>`` otherwise (consumer). Build
+paths off the returned root so the same script resolves correctly in both
+contexts without a code change.
+
+Refs #1875 (content/ move), #1669 (Wave-1 LockedDecisions C1 flatten).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+CONTENT_DIRNAME = "content"
+
+
+def content_root(framework_root: Path | str) -> Path:
+    """Return the directory that holds flattened shippable content.
+
+    ``framework_root`` is the directory that owns ``scripts/`` (i.e. two
+    parents up from a ``scripts/<x>.py`` module). In a source checkout the
+    shippable content lives under ``<framework_root>/content``; in a consumer
+    deposit the C1 flatten removed the prefix, so it lives directly under
+    ``<framework_root>``.
+    """
+    root = Path(framework_root)
+    candidate = root / CONTENT_DIRNAME
+    return candidate if candidate.is_dir() else root

package/scripts/_doctor_state.py

@@ -0,0 +1,277 @@
+"""scripts/_doctor_state.py -- doctor throttle state read/write (#1308).
+
+Schema for ``vbrief/.eval/doctor-state.json``::
+
+    {
+      "last_run_at":        "2026-05-22T13:00:00Z",  # UTC ISO-8601, seconds
+      "last_exit_code":     0,
+      "last_finding_count": 0,
+      "last_error_count":   0
+    }
+
+Count semantics (#1316): ``last_finding_count`` is the count of findings
+that *mattered* -- it EXCLUDES ``severity == "skip"`` findings. A skip
+(e.g. doctor's AGENTS.md-freshness check reporting "no managed-section
+markers (likely maintainer repo)") carries neither error nor warning
+weight, so it must not inflate the persisted tally. The caller
+(``scripts/doctor.py::_persist_doctor_state``) is responsible for
+filtering skips before calling :func:`write_state`. This keeps the
+throttle-skip status line correct: ``_render_doctor_status_line``
+derives the warning tally as ``last_finding_count - last_error_count``,
+and a counted skip would over-report warnings by one on a dirty
+throttle-skip.
+
+Throttle rules:
+
+* 24h after a clean previous run (``last_error_count == 0``).
+* 4h after a dirty previous run (``last_error_count > 0``).
+* Warnings alone count toward the 24h window so stable-warning installs
+  (consumer without ``node`` who does not need it, etc.) are not
+  perpetually re-probed.
+* Corrupt state file (malformed JSON / missing keys / bad types) is
+  treated as no-state -- the caller runs a full check.
+
+The default state-file path is ``<project_root>/vbrief/.eval/doctor-state.json``.
+Tests and other callers MAY set the ``DEFT_DOCTOR_STATE_PATH`` environment
+variable to redirect the path -- this is the seam ``tests/cli/test_doctor_throttle.py``
+uses to isolate per-test state without touching the live framework
+checkout's state file.
+
+Pure stdlib. Best-effort: read / write helpers NEVER raise; they
+silently degrade to the no-state path so the doctor itself never breaks
+because of a state-file bug.
+
+Story: #1308 -- consolidated ``run doctor`` + ``task doctor`` throttle.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+STATE_FILENAME = "doctor-state.json"
+STATE_PARENT = Path("vbrief") / ".eval"
+
+CLEAN_WINDOW_HOURS = 24
+DIRTY_WINDOW_HOURS = 4
+
+_ENV_STATE_PATH = "DEFT_DOCTOR_STATE_PATH"
+
+
+@dataclass(frozen=True)
+class DoctorState:
+    """Parsed doctor-state.json payload (UTC-aware)."""
+
+    last_run_at: datetime
+    last_exit_code: int
+    last_finding_count: int
+    last_error_count: int
+
+
+@dataclass(frozen=True)
+class ThrottleDecision:
+    """Result of :func:`decide_throttle`.
+
+    ``skip`` is True when the throttle window has not yet expired AND the
+    caller has not bypassed the gate (``--full``). ``dirty`` distinguishes
+    the clean (``last_error_count == 0``) and dirty (``last_error_count > 0``)
+    branches so the CLI emits the right status line and exit code.
+
+    When ``state`` is ``None`` (first run / corrupt state file), ``skip``
+    is False and every numeric field is 0; ``last_run_at`` and
+    ``next_eligible_at`` are ``None``.
+    """
+
+    skip: bool
+    dirty: bool
+    last_run_at: datetime | None
+    last_exit_code: int
+    last_finding_count: int
+    last_error_count: int
+    next_eligible_at: datetime | None
+    age_hours: float
+
+
+def state_path(project_root: Path) -> Path:
+    """Return the doctor-state.json path for ``project_root``.
+
+    Honors the ``DEFT_DOCTOR_STATE_PATH`` env override so callers (tests,
+    cron jobs, multi-project setups) can redirect the state file without
+    monkeypatching this module.
+    """
+    override = os.environ.get(_ENV_STATE_PATH, "").strip()
+    if override:
+        return Path(override).expanduser()
+    return project_root / STATE_PARENT / STATE_FILENAME
+
+
+def _parse_iso(ts: object) -> datetime | None:
+    """Parse an ISO-8601 timestamp; return None on any malformed input.
+
+    Mirrors ``run::_parse_iso_utc`` so the doctor's throttle parser stays
+    in lockstep with the remote-probe throttle parser without importing
+    ``run`` (which has heavy import-time side effects).
+    """
+    if not isinstance(ts, str) or not ts:
+        return None
+    candidate = ts.strip()
+    if candidate.endswith("Z"):
+        candidate = candidate[:-1] + "+00:00"
+    try:
+        parsed = datetime.fromisoformat(candidate)
+    except (TypeError, ValueError):
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    return parsed.astimezone(UTC)
+
+
+def read_state(project_root: Path) -> DoctorState | None:
+    """Best-effort read of doctor-state.json.
+
+    Returns the parsed :class:`DoctorState` on a well-formed file, or
+    ``None`` on any failure mode (missing file, malformed JSON, missing
+    keys, bad value types). Per the #1308 contract, corrupt state is
+    indistinguishable from no-state -- both routes converge on \"run the
+    full check\".
+    """
+    path = state_path(project_root)
+    if not path.is_file():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    last_run_at = _parse_iso(data.get("last_run_at"))
+    if last_run_at is None:
+        return None
+    try:
+        last_exit_code = int(data.get("last_exit_code", 0))
+        last_finding_count = int(data.get("last_finding_count", 0))
+        last_error_count = int(data.get("last_error_count", 0))
+    except (TypeError, ValueError):
+        return None
+    return DoctorState(
+        last_run_at=last_run_at,
+        last_exit_code=last_exit_code,
+        last_finding_count=last_finding_count,
+        last_error_count=last_error_count,
+    )
+
+
+def write_state(
+    project_root: Path,
+    *,
+    exit_code: int,
+    finding_count: int,
+    error_count: int,
+    now: datetime | None = None,
+) -> Path | None:
+    """Best-effort write of doctor-state.json.
+
+    Returns the persisted path on success, ``None`` on any OSError
+    (read-only filesystem, permission denied, missing parent dir we
+    cannot create, ...). Never raises -- a state-file write failure
+    MUST NOT break the doctor.
+
+    ``now`` is exposed so tests can pin the timestamp without
+    monkeypatching ``datetime.now``.
+    """
+    when = now if now is not None else _now_utc()
+    payload = {
+        "last_run_at": _format_utc_iso(when),
+        "last_exit_code": int(exit_code),
+        "last_finding_count": int(finding_count),
+        "last_error_count": int(error_count),
+    }
+    path = state_path(project_root)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(
+            json.dumps(payload, sort_keys=True, indent=2) + "\n",
+            encoding="utf-8",
+        )
+    except OSError:
+        return None
+    return path
+
+
+def decide_throttle(
+    state: DoctorState | None,
+    *,
+    now: datetime | None = None,
+) -> ThrottleDecision:
+    """Compute whether the doctor's full check should be skipped.
+
+    Pure -- no I/O. The caller resolves the state via :func:`read_state`
+    and feeds it in so tests can build synthetic states without
+    touching the filesystem.
+
+    Window rules (per the #1308 spec):
+
+    * Clean previous (``last_error_count == 0``) -> 24h window.
+    * Dirty previous (``last_error_count > 0``)  -> 4h window.
+
+    ``now < state.last_run_at + window`` => skip. ``--full`` bypass is
+    the caller's responsibility -- this helper makes the rule decision
+    on the state alone so it stays deterministic and unit-testable.
+    """
+    when = now if now is not None else _now_utc()
+    if state is None:
+        return ThrottleDecision(
+            skip=False,
+            dirty=False,
+            last_run_at=None,
+            last_exit_code=0,
+            last_finding_count=0,
+            last_error_count=0,
+            next_eligible_at=None,
+            age_hours=0.0,
+        )
+    is_dirty = state.last_error_count > 0
+    window_hours = DIRTY_WINDOW_HOURS if is_dirty else CLEAN_WINDOW_HOURS
+    eligible_at = state.last_run_at + timedelta(hours=window_hours)
+    age = when - state.last_run_at
+    age_hours = age.total_seconds() / 3600.0
+    skip = when < eligible_at
+    return ThrottleDecision(
+        skip=skip,
+        dirty=is_dirty,
+        last_run_at=state.last_run_at,
+        last_exit_code=state.last_exit_code,
+        last_finding_count=state.last_finding_count,
+        last_error_count=state.last_error_count,
+        next_eligible_at=eligible_at,
+        age_hours=age_hours,
+    )
+
+
+def _now_utc() -> datetime:
+    """UTC-aware ``datetime.now`` (split out so tests can monkeypatch)."""
+    return datetime.now(UTC)
+
+
+def _format_utc_iso(when: datetime) -> str:
+    """Format a UTC datetime as ``YYYY-MM-DDTHH:MM:SSZ``."""
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=UTC)
+    return when.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+__all__ = [
+    "CLEAN_WINDOW_HOURS",
+    "DIRTY_WINDOW_HOURS",
+    "DoctorState",
+    "STATE_FILENAME",
+    "STATE_PARENT",
+    "ThrottleDecision",
+    "decide_throttle",
+    "read_state",
+    "state_path",
+    "write_state",
+]