@event4u/agent-config 1.13.0 → 1.15.0

package/.agent-src/templates/scripts/telemetry/report_renderer.py

@@ -0,0 +1,170 @@
+"""Engagement report renderer (Phase 4 Step 2).
+
+Two output formats sharing one quartile-bucketing pass:
+
+- **markdown** — human-friendly table grouped into Essential (top 20 %),
+  Useful (mid 60 %), Retirement candidates (bottom 20 %).
+- **json** — machine-readable summary; the same buckets, plus the
+  raw aggregate metadata, so downstream tooling never re-parses
+  the JSONL.
+
+The bucketing is rank-based on ``applied`` count (the signal we care
+about). Ties keep the deterministic order from
+``aggregator.rank_artefacts``. Empty inputs yield an empty-but-valid
+report — the renderer never raises on an empty log.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from .aggregator import AggregateResult, ArtefactStat, rank_artefacts
+from .engagement import check_id_redaction
+
+QUARTILE_TOP_RATIO = 0.20
+QUARTILE_BOTTOM_RATIO = 0.20
+
+BUCKET_TOP = "essential"
+BUCKET_MID = "useful"
+BUCKET_BOTTOM = "retirement_candidate"
+
+
+@dataclass(frozen=True)
+class BucketedStat:
+    stat: ArtefactStat
+    bucket: str
+
+
+def bucketise(stats: Sequence[ArtefactStat]) -> list[BucketedStat]:
+    """Assign each stat to a quartile bucket.
+
+    Rank-based: indices ``[0, top_cut)`` → essential,
+    ``[top_cut, bottom_cut)`` → useful, ``[bottom_cut, n)`` →
+    retirement-candidate. Ranking from ``rank_artefacts`` is assumed.
+
+    For very small samples the cuts collapse:
+    n <= 1   → everything is essential
+    n <= 4   → top 1 essential, rest useful, none retirement
+    n  >= 5  → at least 1 in each bucket
+    """
+    n = len(stats)
+    if n == 0:
+        return []
+    if n <= 1:
+        return [BucketedStat(stat=stats[0], bucket=BUCKET_TOP)]
+    top_cut = max(1, int(round(n * QUARTILE_TOP_RATIO)))
+    bottom_cut = n - max(1, int(round(n * QUARTILE_BOTTOM_RATIO))) if n >= 5 else n
+    if bottom_cut <= top_cut:
+        bottom_cut = n  # mid takes the rest, no retirement bucket
+    out: list[BucketedStat] = []
+    for idx, stat in enumerate(stats):
+        if idx < top_cut:
+            bucket = BUCKET_TOP
+        elif idx < bottom_cut:
+            bucket = BUCKET_MID
+        else:
+            bucket = BUCKET_BOTTOM
+        out.append(BucketedStat(stat=stat, bucket=bucket))
+    return out
+
+
+def render_markdown(
+    aggregate: AggregateResult,
+    *,
+    top: int | None = None,
+    since_label: str | None = None,
+) -> str:
+    """Render a markdown report. ``top`` truncates each bucket; ``None`` keeps all."""
+    ranked = rank_artefacts(aggregate.stats())
+    bucketed = bucketise(ranked)
+    grouped: dict[str, list[BucketedStat]] = {BUCKET_TOP: [], BUCKET_MID: [], BUCKET_BOTTOM: []}
+    for entry in bucketed:
+        grouped[entry.bucket].append(entry)
+
+    lines: list[str] = []
+    lines.append("# Artefact Engagement Report")
+    lines.append("")
+    lines.append(f"- events parsed: **{aggregate.parsed_events}**")
+    lines.append(f"- events skipped (malformed): **{aggregate.skipped_lines}**")
+    if since_label:
+        lines.append(f"- window: **{since_label}**")
+    if aggregate.earliest_ts and aggregate.latest_ts:
+        lines.append(f"- ts range: `{aggregate.earliest_ts}` → `{aggregate.latest_ts}`")
+    lines.append("")
+
+    titles = {
+        BUCKET_TOP: "Essential (top 20 %)",
+        BUCKET_MID: "Useful (mid 60 %)",
+        BUCKET_BOTTOM: "Retirement candidates (bottom 20 %)",
+    }
+    for bucket in (BUCKET_TOP, BUCKET_MID, BUCKET_BOTTOM):
+        rows = grouped[bucket]
+        if top is not None:
+            rows = rows[:top]
+        lines.append(f"## {titles[bucket]}")
+        lines.append("")
+        if not rows:
+            lines.append("_(none)_")
+            lines.append("")
+            continue
+        lines.append("| kind | id | consulted | applied | applied/consulted | last seen |")
+        lines.append("|---|---|---:|---:|---:|---|")
+        for entry in rows:
+            s = entry.stat
+            # Phase 5 export gate — applies to markdown too, not just JSON.
+            check_id_redaction(f"buckets.{s.kind}.id", s.artefact_id)
+            lines.append(
+                f"| {s.kind} | `{s.artefact_id}` | {s.consulted} | {s.applied} "
+                f"| {s.applied_ratio:.2f} | `{s.last_seen_ts}` |"
+            )
+        lines.append("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def render_json(
+    aggregate: AggregateResult,
+    *,
+    top: int | None = None,
+    since_label: str | None = None,
+) -> str:
+    ranked = rank_artefacts(aggregate.stats())
+    bucketed = bucketise(ranked)
+    grouped: dict[str, list[dict[str, Any]]] = {BUCKET_TOP: [], BUCKET_MID: [], BUCKET_BOTTOM: []}
+    for entry in bucketed:
+        grouped[entry.bucket].append(_stat_to_dict(entry.stat))
+    if top is not None:
+        for bucket in grouped:
+            grouped[bucket] = grouped[bucket][:top]
+    payload = {
+        "schema_version": 1,
+        "summary": {
+            "parsed_events": aggregate.parsed_events,
+            "skipped_lines": aggregate.skipped_lines,
+            "total_events": aggregate.total_events,
+            "earliest_ts": aggregate.earliest_ts,
+            "latest_ts": aggregate.latest_ts,
+            "since_label": since_label,
+        },
+        "buckets": grouped,
+    }
+    return json.dumps(payload, sort_keys=True, indent=2) + "\n"
+
+
+def _stat_to_dict(stat: ArtefactStat) -> dict[str, Any]:
+    # Phase 5 export gate: every id leaving the renderer is re-validated
+    # against the same redaction floor that the schema enforces on
+    # write. A pre-validator log (or one hand-edited offline) can never
+    # leak path-shaped or free-text content into a shared report.
+    check_id_redaction(f"buckets.{stat.kind}.id", stat.artefact_id)
+    return {
+        "kind": stat.kind,
+        "id": stat.artefact_id,
+        "consulted": stat.consulted,
+        "applied": stat.applied,
+        "applied_ratio": round(stat.applied_ratio, 4),
+        "last_seen_ts": stat.last_seen_ts,
+    }
+
+
+__all__ = ["BucketedStat", "bucketise", "render_markdown", "render_json"]

package/.agent-src/templates/scripts/telemetry/settings.py

@@ -0,0 +1,112 @@
+"""Shared settings reader for the ``telemetry:*`` CLI commands.
+
+Reads the ``telemetry.artifact_engagement`` namespace from
+``.agent-settings.yml``. Tolerates a missing file, a missing section,
+and missing PyYAML — the default-off doctrine means "everything
+unparseable means disabled".
+
+Single source of truth so ``telemetry_record.py`` and
+``telemetry_status.py`` cannot drift on what counts as "enabled".
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+DEFAULT_LOG_PATH = Path(".agent-engagement.jsonl")
+DEFAULT_GRANULARITY = "task"
+ALLOWED_GRANULARITIES = ("task", "phase-step", "tool-call")
+
+
+@dataclass(frozen=True)
+class TelemetrySettings:
+    enabled: bool
+    granularity: str
+    log_path: Path
+    record_consulted: bool
+    record_applied: bool
+
+    @property
+    def section_present(self) -> bool:
+        # Distinguishes "disabled because section absent" from
+        # "disabled because someone wrote enabled: false". The status
+        # CLI uses this to render a different hint.
+        return self._section_present  # type: ignore[attr-defined]
+
+
+def _coerce_bool(value: Any, default: bool) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalised = value.strip().lower()
+        if normalised in ("true", "yes", "on", "1"):
+            return True
+        if normalised in ("false", "no", "off", "0"):
+            return False
+    return default
+
+
+def _coerce_str(value: Any, default: str, allowed: tuple[str, ...] | None = None) -> str:
+    if not isinstance(value, str) or not value.strip():
+        return default
+    candidate = value.strip()
+    if allowed and candidate not in allowed:
+        return default
+    return candidate
+
+
+def _coerce_path(value: Any, default: Path) -> Path:
+    if not isinstance(value, str) or not value.strip():
+        return default
+    return Path(value.strip())
+
+
+def read_settings(path: Path) -> TelemetrySettings:
+    """Return parsed telemetry settings — never raises on missing data."""
+    section: dict[str, Any] = {}
+    section_present = False
+
+    if path.is_file():
+        try:
+            import yaml  # type: ignore[import-not-found]
+        except ImportError:
+            yaml = None  # type: ignore[assignment]
+        if yaml is not None:
+            try:
+                raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+            except Exception:
+                raw = {}
+            if isinstance(raw, dict):
+                tele = raw.get("telemetry")
+                if isinstance(tele, dict):
+                    artefact = tele.get("artifact_engagement")
+                    if isinstance(artefact, dict):
+                        section = artefact
+                        section_present = True
+
+    record = section.get("record") if isinstance(section.get("record"), dict) else {}
+    output = section.get("output") if isinstance(section.get("output"), dict) else {}
+
+    settings = TelemetrySettings(
+        enabled=_coerce_bool(section.get("enabled"), default=False),
+        granularity=_coerce_str(
+            section.get("granularity"),
+            default=DEFAULT_GRANULARITY,
+            allowed=ALLOWED_GRANULARITIES,
+        ),
+        log_path=_coerce_path(output.get("path"), DEFAULT_LOG_PATH),
+        record_consulted=_coerce_bool(record.get("consulted"), default=True),
+        record_applied=_coerce_bool(record.get("applied"), default=True),
+    )
+    # Carry the section-present flag without breaking dataclass frozen-ness.
+    object.__setattr__(settings, "_section_present", section_present)
+    return settings
+
+
+__all__ = [
+    "DEFAULT_GRANULARITY",
+    "DEFAULT_LOG_PATH",
+    "TelemetrySettings",
+    "read_settings",
+]

package/.agent-src/templates/scripts/telemetry_record.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""``./agent-config telemetry:record`` — append one engagement event.
+
+Reads the ``telemetry.artifact_engagement`` namespace from
+``.agent-settings.yml``. When ``enabled: false`` (default) the script
+exits 0 silently and performs zero file IO — the default-off doctrine
+for this whole feature.
+
+Usage:
+    # JSON payload via --payload-file (consumed atomically)
+    ./agent-config telemetry:record --payload-file payload.json
+
+    # JSON payload via stdin
+    cat payload.json | ./agent-config telemetry:record --stdin
+
+    # Direct construction (idempotent within boundary if reused via
+    # the BoundarySession class — at the CLI layer, each call writes
+    # one line)
+    ./agent-config telemetry:record \\
+        --task-id ticket-PROJ-42 --boundary task \\
+        --consulted skills:php-coder --consulted rules:scope-control \\
+        --applied skills:php-coder
+
+Exit codes:
+    0   success or disabled (silent)
+    1   schema-validation failure
+    2   IO / settings parse error
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# Resolve sibling ``telemetry/`` package — Python adds the script's
+# directory to sys.path automatically, so this import works whether
+# the script is dispatched from the package or from a consumer copy.
+from telemetry.boundary import record_event
+from telemetry.engagement import (
+    EngagementEvent,
+    EngagementSchemaError,
+    now_utc_iso,
+)
+from telemetry.settings import TelemetrySettings, read_settings
+
+
+def _parse_kv_list(values: list[str]) -> dict[str, list[str]]:
+    """Turn ``["skills:a", "skills:b", "rules:c"]`` into a kind→ids dict."""
+    out: dict[str, list[str]] = {}
+    for raw in values:
+        if ":" not in raw:
+            raise SystemExit(
+                f"❌  --consulted/--applied must be 'kind:id', got {raw!r}"
+            )
+        kind, _, art_id = raw.partition(":")
+        kind = kind.strip()
+        art_id = art_id.strip()
+        if not kind or not art_id:
+            raise SystemExit(
+                f"❌  empty kind or id in {raw!r}"
+            )
+        out.setdefault(kind, []).append(art_id)
+    return out
+
+
+def _build_event_from_args(args: argparse.Namespace) -> EngagementEvent:
+    return EngagementEvent(
+        ts=args.ts or now_utc_iso(),
+        task_id=args.task_id,
+        boundary_kind=args.boundary,
+        consulted=_parse_kv_list(args.consulted or []),
+        applied=_parse_kv_list(args.applied or []),
+    )
+
+
+def _build_event_from_payload(raw: str) -> EngagementEvent:
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise SystemExit(f"❌  payload is not valid JSON: {exc}")
+    if not isinstance(data, dict):
+        raise SystemExit("❌  payload must be a JSON object")
+    return EngagementEvent(
+        ts=data.get("ts") or now_utc_iso(),
+        task_id=data.get("task_id", ""),
+        boundary_kind=data.get("boundary_kind", ""),
+        consulted=data.get("consulted", {}) or {},
+        applied=data.get("applied", {}) or {},
+        tokens_estimate=data.get("tokens_estimate"),
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="agent-config telemetry:record",
+        description=(
+            "Append one artefact-engagement event to the local JSONL log. "
+            "Default-off — silent exit 0 unless explicitly enabled."
+        ),
+    )
+    parser.add_argument("--task-id", default="")
+    parser.add_argument(
+        "--boundary",
+        default="task",
+        choices=("task", "phase-step", "tool-call"),
+    )
+    parser.add_argument("--consulted", action="append")
+    parser.add_argument("--applied", action="append")
+    parser.add_argument("--ts", default="")
+    parser.add_argument("--payload-file", type=Path)
+    parser.add_argument("--stdin", action="store_true")
+    parser.add_argument(
+        "--settings",
+        type=Path,
+        default=Path(".agent-settings.yml"),
+        help="Override settings path (tests).",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Bypass the enabled-flag (tests + maintainer one-shots).",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        settings: TelemetrySettings = read_settings(args.settings)
+    except OSError as exc:
+        print(f"❌  cannot read settings: {exc}", file=sys.stderr)
+        return 2
+
+    if not settings.enabled and not args.force:
+        # Default-off: silent success. Crucially: no payload parsing,
+        # no schema construction — zero work attributable to telemetry.
+        return 0
+
+    if args.payload_file:
+        try:
+            raw = args.payload_file.read_text(encoding="utf-8")
+        except OSError as exc:
+            print(f"❌  cannot read --payload-file: {exc}", file=sys.stderr)
+            return 2
+        event = _build_event_from_payload(raw)
+    elif args.stdin:
+        event = _build_event_from_payload(sys.stdin.read())
+    else:
+        if not args.task_id:
+            print("❌  --task-id required (or pass --payload-file/--stdin)",
+                  file=sys.stderr)
+            return 1
+        event = _build_event_from_args(args)
+
+    try:
+        record_event(settings.log_path, event)
+    except EngagementSchemaError as exc:
+        print(f"❌  schema validation failed: {exc}", file=sys.stderr)
+        return 1
+    except OSError as exc:
+        print(f"❌  cannot write engagement log: {exc}", file=sys.stderr)
+        return 2
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.agent-src/templates/scripts/telemetry_report.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""``./agent-config telemetry:report`` — aggregate the engagement log.
+
+Reads ``.agent-engagement.jsonl``, groups events by ``(kind, id)``,
+quartile-buckets the artefacts (essential / useful / retirement
+candidate), and prints a markdown table or JSON document.
+
+Usage:
+    # Defaults: --since 30d --top 20 markdown
+    ./agent-config telemetry:report
+
+    # Last 7 days, JSON, no truncation
+    ./agent-config telemetry:report --since 7d --format json --top 0
+
+    # Override log path (tests; consumer copies; reports on archived logs)
+    ./agent-config telemetry:report --log-path /tmp/snapshot.jsonl
+
+Exit codes:
+    0   success (empty log → empty-but-valid report)
+    2   IO / settings parse error, unparseable --since, or
+        redaction-validator failure on a row sourced from the log
+        (a path-shaped or extension-shaped id slipped past the write
+        gate; the report is refused rather than shared)
+"""
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from telemetry.aggregator import aggregate
+from telemetry.engagement import EngagementSchemaError
+from telemetry.report_renderer import render_json, render_markdown
+from telemetry.settings import read_settings
+
+_DURATION_RE = re.compile(r"^\s*(\d+)\s*([dhm])\s*$")
+
+
+def _parse_since(value: str | None) -> tuple[datetime | None, str | None]:
+    """Parse ``30d`` / ``7d`` / ``24h`` / ``60m`` into a UTC cutoff.
+
+    Returns ``(cutoff_or_None, human_label)``. ``value`` of ``None`` or
+    ``"all"`` means "no lower bound" — both cutoff and label are
+    ``None``. Raises ``ValueError`` on malformed input so the CLI can
+    surface a clean error and exit 2.
+    """
+    if value is None or value.strip().lower() == "all":
+        return None, None
+    match = _DURATION_RE.match(value)
+    if not match:
+        raise ValueError(
+            f"--since must be <int>{{d,h,m}} or 'all', got {value!r}"
+        )
+    qty = int(match.group(1))
+    unit = match.group(2)
+    delta = {
+        "d": timedelta(days=qty),
+        "h": timedelta(hours=qty),
+        "m": timedelta(minutes=qty),
+    }[unit]
+    cutoff = datetime.now(tz=timezone.utc) - delta
+    label = f"last {qty}{unit}"
+    return cutoff, label
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="agent-config telemetry:report",
+        description=(
+            "Render an artefact-engagement report from the JSONL log. "
+            "Read-only; never mutates settings or the log."
+        ),
+    )
+    parser.add_argument(
+        "--since",
+        default="30d",
+        help=(
+            "Lower-bound time window: <int>{d,h,m} or 'all'. "
+            "Default: 30d. Events at or before the cutoff are excluded."
+        ),
+    )
+    parser.add_argument(
+        "--top",
+        type=int,
+        default=20,
+        help=(
+            "Truncate each bucket to N rows. Default: 20. Use 0 to "
+            "disable truncation."
+        ),
+    )
+    parser.add_argument(
+        "--format",
+        choices=("markdown", "json"),
+        default="markdown",
+    )
+    parser.add_argument(
+        "--log-path",
+        type=Path,
+        default=None,
+        help=(
+            "Override the log path. Default: read "
+            "telemetry.artifact_engagement.output.path from "
+            ".agent-settings.yml (falls back to .agent-engagement.jsonl)."
+        ),
+    )
+    parser.add_argument(
+        "--settings",
+        type=Path,
+        default=Path(".agent-settings.yml"),
+        help="Override settings path (tests).",
+    )
+    args = parser.parse_args(argv)
+
+    try:
+        cutoff, since_label = _parse_since(args.since)
+    except ValueError as exc:
+        print(f"❌  {exc}", file=sys.stderr)
+        return 2
+
+    if args.log_path is not None:
+        log_path = args.log_path
+    else:
+        try:
+            settings = read_settings(args.settings)
+        except OSError as exc:
+            print(f"❌  cannot read settings: {exc}", file=sys.stderr)
+            return 2
+        log_path = settings.log_path
+
+    try:
+        result = aggregate(log_path, since=cutoff)
+    except OSError as exc:
+        print(f"❌  cannot read log {log_path}: {exc}", file=sys.stderr)
+        return 2
+
+    top = None if args.top <= 0 else args.top
+    try:
+        if args.format == "json":
+            rendered = render_json(result, top=top, since_label=since_label)
+        else:
+            rendered = render_markdown(result, top=top, since_label=since_label)
+    except EngagementSchemaError as exc:
+        print(
+            f"❌  redaction validator refused report: {exc}",
+            file=sys.stderr,
+        )
+        return 2
+    sys.stdout.write(rendered)
+
+    if result.skipped_lines:
+        print(
+            f"⚠️   skipped {result.skipped_lines} malformed line(s)",
+            file=sys.stderr,
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())