@event4u/agent-config 1.12.0 → 1.14.0

package/.agent-src/templates/scripts/telemetry/__init__.py

@@ -0,0 +1,42 @@
+"""``telemetry`` — artefact engagement recording (default-off).
+
+The package owns the local-only engagement log
+(``.agent-engagement.jsonl``) that records, at task boundaries, which
+artefacts (skills, rules, commands, guidelines, personas) the agent
+``consulted`` (loaded into context) and ``applied`` (cited or directly
+drove a decision).
+
+Architectural constraints (from
+``agents/roadmaps/road-to-artifact-engagement-telemetry.md`` Phase 1):
+
+- Default-off. ``telemetry.artifact_engagement.enabled: false`` in
+  ``.agent-settings.yml`` produces zero file IO and zero token cost.
+- Local only. No server-side aggregation, no cross-repo sync.
+- ID-only payloads. No paths, no file contents, no prompts, no
+  secrets ever reach the log.
+- Append-only JSONL. One event per task / phase-step boundary.
+- Strict schema. Unknown artefact kinds are rejected.
+"""
+from __future__ import annotations
+
+from .engagement import (
+    ALLOWED_BOUNDARY_KINDS,
+    ALLOWED_KINDS,
+    SCHEMA_VERSION,
+    EngagementEvent,
+    EngagementSchemaError,
+    append_event,
+    now_utc_iso,
+    parse_event,
+)
+
+__all__ = [
+    "ALLOWED_BOUNDARY_KINDS",
+    "ALLOWED_KINDS",
+    "SCHEMA_VERSION",
+    "EngagementEvent",
+    "EngagementSchemaError",
+    "append_event",
+    "now_utc_iso",
+    "parse_event",
+]

package/.agent-src/templates/scripts/telemetry/aggregator.py

@@ -0,0 +1,154 @@
+"""Engagement-log aggregator (Phase 4).
+
+Pure-stdlib reader: streams ``.agent-engagement.jsonl``, groups events by
+``(kind, id)``, and returns per-artefact statistics. The renderer in
+``report_renderer.py`` consumes the dataclasses produced here.
+
+Design contract:
+
+- **Skip, don't crash.** Malformed JSONL lines are counted in
+  ``AggregateResult.skipped_lines`` and dropped. Phase 4 Step 4 locks
+  this behaviour: a single corrupt line in a 10k-line log must not
+  block the report.
+- **No IO besides the log read.** No network, no settings reads, no
+  log creation. Caller (CLI) is responsible for feeding a real path.
+- **``since`` is exclusive on the lower bound** — ``since`` of
+  ``2026-04-01T00:00:00Z`` keeps events with ``ts > since``. ``None``
+  means "include everything".
+- **Stats are sort-stable.** ``rank_artefacts`` returns a list ordered
+  by ``applied`` desc, ``consulted`` desc, then ``(kind, id)`` asc, so
+  two reports over the same log render byte-identical.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable, Iterator
+
+from .engagement import EngagementEvent, EngagementSchemaError, parse_event
+
+
+@dataclass(frozen=True)
+class ArtefactStat:
+    kind: str
+    artefact_id: str
+    consulted: int
+    applied: int
+    last_seen_ts: str
+
+    @property
+    def applied_ratio(self) -> float:
+        """Applied / consulted. ``0.0`` when never consulted (impossible
+        in practice — applied is a strict subset of consulted — but the
+        guard keeps the division safe for malformed inputs)."""
+        return (self.applied / self.consulted) if self.consulted else 0.0
+
+
+@dataclass
+class AggregateResult:
+    total_events: int = 0
+    parsed_events: int = 0
+    skipped_lines: int = 0
+    earliest_ts: str | None = None
+    latest_ts: str | None = None
+    artefacts: dict[tuple[str, str], dict[str, object]] = field(default_factory=dict)
+
+    def stats(self) -> list[ArtefactStat]:
+        """Materialise the accumulated buckets as immutable stats."""
+        out: list[ArtefactStat] = []
+        for (kind, art_id), bucket in self.artefacts.items():
+            out.append(
+                ArtefactStat(
+                    kind=kind,
+                    artefact_id=art_id,
+                    consulted=int(bucket["consulted"]),
+                    applied=int(bucket["applied"]),
+                    last_seen_ts=str(bucket["last_seen_ts"]),
+                )
+            )
+        return out
+
+
+def _parse_iso(ts: str) -> datetime | None:
+    """Parse a ``%Y-%m-%dT%H:%M:%SZ`` stamp into UTC. Returns ``None``
+    for malformed stamps so the caller can skip the comparison cleanly.
+    """
+    if not isinstance(ts, str) or not ts:
+        return None
+    try:
+        # strptime with literal Z handles the ``now_utc_iso`` format.
+        return datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
+    except ValueError:
+        return None
+
+
+def _iter_events(log_path: Path) -> Iterator[tuple[int, EngagementEvent | None]]:
+    """Yield ``(line_number, event_or_None)``. ``None`` signals a skip."""
+    if not log_path.is_file():
+        return
+    with log_path.open("r", encoding="utf-8") as fh:
+        for line_no, line in enumerate(fh, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                event = parse_event(stripped + "\n")
+            except EngagementSchemaError:
+                yield line_no, None
+                continue
+            yield line_no, event
+
+
+def aggregate(
+    log_path: Path,
+    *,
+    since: datetime | None = None,
+) -> AggregateResult:
+    """Stream the JSONL log and compute per-artefact stats."""
+    result = AggregateResult()
+    for _line_no, event in _iter_events(log_path):
+        result.total_events += 1
+        if event is None:
+            result.skipped_lines += 1
+            continue
+        ts = _parse_iso(event.ts)
+        if since is not None and ts is not None and ts <= since:
+            continue
+        result.parsed_events += 1
+        if result.earliest_ts is None or event.ts < result.earliest_ts:
+            result.earliest_ts = event.ts
+        if result.latest_ts is None or event.ts > result.latest_ts:
+            result.latest_ts = event.ts
+        _accumulate(result.artefacts, event.consulted, event.applied, event.ts)
+    return result
+
+
+def _accumulate(
+    bucket: dict[tuple[str, str], dict[str, object]],
+    consulted: dict[str, list[str]],
+    applied: dict[str, list[str]],
+    ts: str,
+) -> None:
+    for kind, ids in consulted.items():
+        for art_id in ids:
+            entry = bucket.setdefault((kind, art_id), {"consulted": 0, "applied": 0, "last_seen_ts": ""})
+            entry["consulted"] = int(entry["consulted"]) + 1  # type: ignore[operator]
+            if ts > str(entry["last_seen_ts"]):
+                entry["last_seen_ts"] = ts
+    for kind, ids in applied.items():
+        for art_id in ids:
+            entry = bucket.setdefault((kind, art_id), {"consulted": 0, "applied": 0, "last_seen_ts": ""})
+            entry["applied"] = int(entry["applied"]) + 1  # type: ignore[operator]
+            if ts > str(entry["last_seen_ts"]):
+                entry["last_seen_ts"] = ts
+
+
+def rank_artefacts(stats: Iterable[ArtefactStat]) -> list[ArtefactStat]:
+    return sorted(
+        stats,
+        key=lambda s: (-s.applied, -s.consulted, s.kind, s.artefact_id),
+    )
+
+
+__all__ = ["ArtefactStat", "AggregateResult", "aggregate", "rank_artefacts"]

package/.agent-src/templates/scripts/telemetry/boundary.py

@@ -0,0 +1,171 @@
+"""Boundary detection + concurrent-safe recording (Phase 2).
+
+Two responsibilities, one module:
+
+1. ``BoundarySession`` — in-process coalescing. Multiple ``add_*`` calls
+   within one task / phase-step / tool-call boundary merge into a
+   single emitted event (set-union on ``consulted`` / ``applied``).
+   Idempotent: calling ``flush()`` twice without new additions is a
+   no-op; calling ``add_consulted("skills", ["x"])`` twice records
+   ``"x"`` once.
+
+2. ``record_event`` — cross-process durability. Uses ``fcntl.flock``
+   (POSIX) so concurrent writers from separate ``./agent-config
+   telemetry:record`` invocations cannot interleave inside one JSONL
+   line. On non-POSIX (no ``fcntl``) we fall back to a best-effort
+   append; the package only ships on POSIX-compatible CI today.
+
+The CLI in ``cli.py`` is the only caller that should touch the log
+path directly. Agent-side flows wire through ``BoundarySession``.
+"""
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable, Iterator
+
+try:  # POSIX advisory file locking
+    import fcntl  # type: ignore[import-not-found]
+    _HAS_FCNTL = True
+except ImportError:  # pragma: no cover — Windows / sandbox
+    _HAS_FCNTL = False
+
+from .engagement import (
+    ALLOWED_BOUNDARY_KINDS,
+    ALLOWED_KINDS,
+    EngagementEvent,
+    EngagementSchemaError,
+    now_utc_iso,
+)
+
+
+@dataclass
+class BoundarySession:
+    """Collect artefact engagements for one boundary, flush once.
+
+    Use as a context manager — ``__exit__`` flushes on clean exit and
+    suppresses on exception (so failed tasks don't pollute the log).
+    """
+
+    task_id: str
+    boundary_kind: str
+    log_path: Path
+    consulted: dict[str, set[str]] = field(default_factory=dict)
+    applied: dict[str, set[str]] = field(default_factory=dict)
+    _flushed: bool = False
+    _has_data: bool = False
+
+    def __post_init__(self) -> None:
+        if self.boundary_kind not in ALLOWED_BOUNDARY_KINDS:
+            raise EngagementSchemaError(
+                f"boundary_kind must be one of {ALLOWED_BOUNDARY_KINDS!r}"
+            )
+        if not isinstance(self.task_id, str) or not self.task_id:
+            raise EngagementSchemaError("task_id must be a non-empty string")
+
+    def add_consulted(self, kind: str, ids: Iterable[str]) -> None:
+        self._merge(self.consulted, kind, ids)
+
+    def add_applied(self, kind: str, ids: Iterable[str]) -> None:
+        self._merge(self.applied, kind, ids)
+
+    def _merge(self, bucket: dict[str, set[str]], kind: str, ids: Iterable[str]) -> None:
+        if kind not in ALLOWED_KINDS:
+            raise EngagementSchemaError(
+                f"{kind!r} is not an allowed artefact kind "
+                f"(allowed: {ALLOWED_KINDS!r})"
+            )
+        target = bucket.setdefault(kind, set())
+        for art_id in ids:
+            if not isinstance(art_id, str) or not art_id:
+                raise EngagementSchemaError(
+                    f"{kind} ids must be non-empty strings"
+                )
+            target.add(art_id)
+            self._has_data = True
+
+    def to_event(self) -> EngagementEvent:
+        return EngagementEvent(
+            ts=now_utc_iso(),
+            task_id=self.task_id,
+            boundary_kind=self.boundary_kind,
+            consulted={k: sorted(v) for k, v in self.consulted.items() if v},
+            applied={k: sorted(v) for k, v in self.applied.items() if v},
+        )
+
+    def flush(self) -> bool:
+        """Write one merged event to the log. Returns True if written.
+
+        No-op when already flushed or no data was added — keeps the
+        boundary idempotent.
+        """
+        if self._flushed or not self._has_data:
+            return False
+        record_event(self.log_path, self.to_event())
+        self._flushed = True
+        return True
+
+    def __enter__(self) -> "BoundarySession":
+        return self
+
+    def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
+        if exc_type is None:
+            self.flush()
+        # On exception: do nothing — failed boundary, no record.
+
+
+def record_event(log_path: Path, event: EngagementEvent) -> None:
+    """Append one event under an exclusive file lock.
+
+    The lock guarantees that two concurrent writers append two
+    complete, well-formed lines instead of one interleaved line. We
+    open with ``"a"`` so each write atomically extends EOF on POSIX
+    once the lock is held.
+    """
+    event.validate()
+    payload = event.to_jsonl().encode("utf-8")
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644)
+    try:
+        if _HAS_FCNTL:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+        try:
+            written = 0
+            while written < len(payload):
+                written += os.write(fd, payload[written:])
+            os.fsync(fd)
+        finally:
+            if _HAS_FCNTL:
+                fcntl.flock(fd, fcntl.LOCK_UN)
+    finally:
+        os.close(fd)
+
+
+@contextmanager
+def open_boundary(
+    task_id: str,
+    boundary_kind: str,
+    log_path: Path,
+) -> Iterator[BoundarySession]:
+    """Convenience context manager around ``BoundarySession``.
+
+    >>> with open_boundary("ticket-1", "task", Path(".agent-engagement.jsonl")) as s:
+    ...     s.add_consulted("skills", ["php-coder"])
+    ...     s.add_applied("skills", ["php-coder"])
+    """
+    session = BoundarySession(
+        task_id=task_id,
+        boundary_kind=boundary_kind,
+        log_path=log_path,
+    )
+    with session:
+        yield session
+
+
+__all__ = [
+    "BoundarySession",
+    "open_boundary",
+    "record_event",
+]

package/.agent-src/templates/scripts/telemetry/engagement.py

@@ -0,0 +1,238 @@
+"""Engagement event schema and JSONL appender (Phase 1).
+
+Stdlib-only. No external deps. All validation is structural — Phase 5
+adds the redaction validator on top. The contract here is:
+
+    {
+      "schema_version": 1,
+      "ts": "<ISO-8601 UTC>",
+      "task_id": "<repo-internal id>",
+      "boundary_kind": "task" | "phase-step" | "tool-call",
+      "consulted": {"skills": [...], "rules": [...], ...},
+      "applied":   {"skills": [...], "rules": [...], ...},
+      "tokens_estimate": {"consulted_load": <int>}   # optional
+    }
+
+Design choices:
+
+- Dataclass + manual ``validate()`` (no pydantic — keep the engine
+  install footprint flat, mirroring ``work_engine``).
+- Append uses ``open(..., "a")`` with ``flush()`` so one record is one
+  line; concurrent-write durability is Phase 2's job (file-lock).
+- ``parse_event`` round-trips a serialised line back into a dataclass
+  for the tests; production agents only ever ``append_event``.
+"""
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+SCHEMA_VERSION = 1
+MAX_ID_LEN = 200
+
+ALLOWED_KINDS: tuple[str, ...] = (
+    "skills",
+    "rules",
+    "commands",
+    "guidelines",
+    "personas",
+)
+
+ALLOWED_BOUNDARY_KINDS: tuple[str, ...] = (
+    "task",
+    "phase-step",
+    "tool-call",
+)
+
+# Phase 5 redaction validator — keep id fields from leaking paths,
+# free-text, or filenames. Repository-internal artefact ids and
+# task ids never contain these characters.
+_FORBIDDEN_ID_CHARS: tuple[str, ...] = ("/", "\\", "\n", "\r", "\t")
+# Trailing alphabetic extension (`.md`, `.py`, `.json`, …). Restricting
+# to alphabetic chars 1-8 long avoids false positives on version-like
+# patterns (e.g. ``v1.0``, ``ticket-1.2``) while still catching the
+# realistic file-extension leak surface.
+_FILE_EXTENSION_RE = re.compile(r"\.[A-Za-z]{1,8}$")
+
+
+class EngagementSchemaError(ValueError):
+    """Raised when an event violates the schema."""
+
+
+def check_id_redaction(label: str, value: str) -> None:
+    """Phase 5 redaction validator — reject path- and free-text-shaped ids.
+
+    Public surface: the schema layer calls this on every ``task_id``
+    and every ``consulted``/``applied`` artefact id before write; the
+    report renderer calls it on every id before emitting JSON, so a
+    pre-validator (or hand-edited) log can never leak into a shared
+    report.
+
+    Caller has already verified ``value`` is a non-empty string of
+    length ``<= MAX_ID_LEN``. This function adds the privacy floor:
+    no slashes, no backslashes, no embedded control chars, no leading
+    or trailing whitespace, no file-extension suffix.
+
+    Failure raises :class:`EngagementSchemaError` with a label that
+    points at the offending field (``task_id``, ``consulted.skills``,
+    ``applied.rules`` …).
+    """
+    if not isinstance(value, str):
+        raise EngagementSchemaError(f"{label} must be a string")
+    if not value:
+        raise EngagementSchemaError(f"{label} must be non-empty")
+    if len(value) > MAX_ID_LEN:
+        raise EngagementSchemaError(
+            f"{label} exceeds {MAX_ID_LEN} chars"
+        )
+    for ch in _FORBIDDEN_ID_CHARS:
+        if ch in value:
+            raise EngagementSchemaError(
+                f"{label} contains forbidden character {ch!r}; "
+                "id fields must be repository-internal artefact ids only "
+                "(no paths, no free-text)"
+            )
+    if value != value.strip():
+        raise EngagementSchemaError(
+            f"{label} must not start or end with whitespace"
+        )
+    if _FILE_EXTENSION_RE.search(value):
+        raise EngagementSchemaError(
+            f"{label} ends in a file extension; "
+            "id fields must be repository-internal artefact ids only "
+            "(strip path + extension before recording)"
+        )
+
+
+@dataclass
+class EngagementEvent:
+    ts: str
+    task_id: str
+    boundary_kind: str
+    consulted: dict[str, list[str]] = field(default_factory=dict)
+    applied: dict[str, list[str]] = field(default_factory=dict)
+    tokens_estimate: dict[str, int] | None = None
+    schema_version: int = SCHEMA_VERSION
+
+    def validate(self) -> None:
+        if not isinstance(self.ts, str) or not self.ts:
+            raise EngagementSchemaError("ts must be a non-empty string")
+        if not isinstance(self.task_id, str) or not self.task_id:
+            raise EngagementSchemaError("task_id must be a non-empty string")
+        if len(self.task_id) > MAX_ID_LEN:
+            raise EngagementSchemaError(
+                f"task_id exceeds {MAX_ID_LEN} chars"
+            )
+        check_id_redaction("task_id", self.task_id)
+        if self.boundary_kind not in ALLOWED_BOUNDARY_KINDS:
+            raise EngagementSchemaError(
+                f"boundary_kind must be one of {ALLOWED_BOUNDARY_KINDS!r}"
+            )
+        _validate_artefact_dict("consulted", self.consulted)
+        _validate_artefact_dict("applied", self.applied)
+        if self.tokens_estimate is not None:
+            if not isinstance(self.tokens_estimate, dict):
+                raise EngagementSchemaError(
+                    "tokens_estimate must be a dict[str,int] or None"
+                )
+            for k, v in self.tokens_estimate.items():
+                if not isinstance(k, str) or not isinstance(v, int):
+                    raise EngagementSchemaError(
+                        "tokens_estimate keys must be str, values int"
+                    )
+        if self.schema_version != SCHEMA_VERSION:
+            raise EngagementSchemaError(
+                f"schema_version must be {SCHEMA_VERSION}, got "
+                f"{self.schema_version!r}"
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        self.validate()
+        out: dict[str, Any] = {
+            "schema_version": self.schema_version,
+            "ts": self.ts,
+            "task_id": self.task_id,
+            "boundary_kind": self.boundary_kind,
+            "consulted": _normalise_artefact_dict(self.consulted),
+            "applied": _normalise_artefact_dict(self.applied),
+        }
+        if self.tokens_estimate:
+            out["tokens_estimate"] = dict(self.tokens_estimate)
+        return out
+
+    def to_jsonl(self) -> str:
+        return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + "\n"
+
+
+def _validate_artefact_dict(label: str, payload: Any) -> None:
+    if not isinstance(payload, dict):
+        raise EngagementSchemaError(f"{label} must be a dict[str,list[str]]")
+    for kind, ids in payload.items():
+        if kind not in ALLOWED_KINDS:
+            raise EngagementSchemaError(
+                f"{label}.{kind!r} is not an allowed artefact kind "
+                f"(allowed: {ALLOWED_KINDS!r})"
+            )
+        if not isinstance(ids, list):
+            raise EngagementSchemaError(
+                f"{label}.{kind} must be a list of str"
+            )
+        for art_id in ids:
+            if not isinstance(art_id, str) or not art_id:
+                raise EngagementSchemaError(
+                    f"{label}.{kind} must contain non-empty str ids"
+                )
+            if len(art_id) > MAX_ID_LEN:
+                raise EngagementSchemaError(
+                    f"{label}.{kind} id exceeds {MAX_ID_LEN} chars"
+                )
+            check_id_redaction(f"{label}.{kind}", art_id)
+
+
+def _normalise_artefact_dict(payload: dict[str, list[str]]) -> dict[str, list[str]]:
+    # Stable shape: only non-empty kinds, ids preserved in declared order.
+    return {kind: list(payload[kind]) for kind in ALLOWED_KINDS if payload.get(kind)}
+
+
+def parse_event(line: str) -> EngagementEvent:
+    if not isinstance(line, str) or not line.strip():
+        raise EngagementSchemaError("line must be a non-empty JSONL record")
+    try:
+        raw = json.loads(line)
+    except json.JSONDecodeError as exc:
+        raise EngagementSchemaError(f"line is not valid JSON: {exc}") from exc
+    if not isinstance(raw, dict):
+        raise EngagementSchemaError("event must be a JSON object")
+    event = EngagementEvent(
+        ts=raw.get("ts", ""),
+        task_id=raw.get("task_id", ""),
+        boundary_kind=raw.get("boundary_kind", ""),
+        consulted=raw.get("consulted", {}) or {},
+        applied=raw.get("applied", {}) or {},
+        tokens_estimate=raw.get("tokens_estimate"),
+        schema_version=raw.get("schema_version", SCHEMA_VERSION),
+    )
+    event.validate()
+    return event
+
+
+def append_event(log_path: Path, event: EngagementEvent) -> None:
+    """Append one validated event to the JSONL log.
+
+    Caller is responsible for the enabled-flag check — this function
+    writes unconditionally. Phase 2 wraps it with the settings probe.
+    """
+    payload = event.to_jsonl()
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    with log_path.open("a", encoding="utf-8") as fh:
+        fh.write(payload)
+        fh.flush()
+
+
+def now_utc_iso() -> str:
+    """ISO-8601 UTC timestamp, second precision, ``Z`` suffix."""
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")