@event4u/agent-config 1.40.0 → 1.41.1

package/scripts/mcp_telemetry_health.py

@@ -0,0 +1,214 @@
+"""MCP telemetry healthcheck — Phase 1 J6.
+
+Asserts that the per-consumer JSONL sink at
+``<consumer_root>/agents/.mcp-telemetry/calls.jsonl`` received at least
+one record inside a configurable window (default 24 h). Exits non-zero
+on silence so the caller's alert sink — Sentry, email, GitHub Actions
+failure, cron mailer — fires.
+
+Per ``agents/roadmaps/archive/road-to-mcp-full-coverage.md`` §Phase 1 J6, the
+healthcheck protects Phase 2 K1 against waking to an empty dataset: a
+silent telemetry pipeline must be visible *during* Phase 1, not after
+the observation window closes.
+
+Usage:
+
+  python3 scripts/mcp_telemetry_health.py                # 24h window
+  python3 scripts/mcp_telemetry_health.py --window-hours 6
+  python3 scripts/mcp_telemetry_health.py --allow-missing  # CI mode
+  python3 scripts/mcp_telemetry_health.py --json           # machine-readable
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+# Re-use the canonical sink location so a contract change in
+# telemetry.py propagates automatically.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from mcp_server.telemetry import (  # noqa: E402
+    TELEMETRY_FILENAME,
+    TELEMETRY_REL_DIR,
+)
+
+DEFAULT_WINDOW_HOURS = 24
+_ISO_FMT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+@dataclass(frozen=True)
+class HealthReport:
+    """Outcome of a single healthcheck run. Serialised when --json fires."""
+
+    status: str  # "healthy" | "silent" | "missing" | "unreadable"
+    path: str
+    window_hours: int
+    records_in_window: int
+    last_ts: str | None
+    message: str
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "status": self.status,
+            "path": self.path,
+            "window_hours": self.window_hours,
+            "records_in_window": self.records_in_window,
+            "last_ts": self.last_ts,
+            "message": self.message,
+        }
+
+
+def _parse_iso(ts: str) -> float | None:
+    """Best-effort ISO-8601 → epoch. Returns None for malformed input."""
+    try:
+        return time.mktime(time.strptime(ts, _ISO_FMT)) - time.timezone
+    except (ValueError, TypeError):
+        return None
+
+
+def resolve_log_path(consumer_root: Path | None = None) -> Path:
+    """Pick the JSONL location — matches telemetry.py's resolver."""
+    root = (consumer_root or Path.cwd()).resolve()
+    return root / TELEMETRY_REL_DIR / TELEMETRY_FILENAME
+
+
+def evaluate(
+    *,
+    consumer_root: Path | None = None,
+    window_hours: int = DEFAULT_WINDOW_HOURS,
+    now: float | None = None,
+) -> HealthReport:
+    """Return a HealthReport — pure function, no exit calls."""
+    target = resolve_log_path(consumer_root)
+    cutoff = (now if now is not None else time.time()) - window_hours * 3600
+
+    if not target.exists():
+        return HealthReport(
+            status="missing",
+            path=str(target),
+            window_hours=window_hours,
+            records_in_window=0,
+            last_ts=None,
+            message=(
+                f"Telemetry sink not found at {target}. "
+                "Either the MCP server has never run, or the consumer root is wrong."
+            ),
+        )
+
+    try:
+        lines = target.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        return HealthReport(
+            status="unreadable",
+            path=str(target),
+            window_hours=window_hours,
+            records_in_window=0,
+            last_ts=None,
+            message=f"Telemetry sink unreadable: {exc}",
+        )
+
+    in_window = 0
+    last_ts: str | None = None
+    for line in lines:
+        if not line.strip():
+            continue
+        try:
+            record = json.loads(line)
+        except ValueError:
+            continue
+        ts = record.get("ts")
+        if not isinstance(ts, str):
+            continue
+        epoch = _parse_iso(ts)
+        if epoch is None:
+            continue
+        if last_ts is None or ts > last_ts:
+            last_ts = ts
+        if epoch >= cutoff:
+            in_window += 1
+
+    if in_window == 0:
+        return HealthReport(
+            status="silent",
+            path=str(target),
+            window_hours=window_hours,
+            records_in_window=0,
+            last_ts=last_ts,
+            message=(
+                f"No telemetry records in the past {window_hours}h. "
+                "Phase 2 K1 dataset is at risk — verify the MCP server is reachable "
+                "and that consumers are calling tools."
+            ),
+        )
+
+    return HealthReport(
+        status="healthy",
+        path=str(target),
+        window_hours=window_hours,
+        records_in_window=in_window,
+        last_ts=last_ts,
+        message=f"{in_window} record(s) logged in the past {window_hours}h.",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description=(
+            "MCP telemetry healthcheck — exits non-zero if no calls were "
+            "logged in the configured window."
+        ),
+    )
+    parser.add_argument(
+        "--consumer-root",
+        type=Path,
+        default=None,
+        help="Root directory containing agents/.mcp-telemetry/ (default: cwd).",
+    )
+    parser.add_argument(
+        "--window-hours",
+        type=int,
+        default=DEFAULT_WINDOW_HOURS,
+        help=f"Hours back to scan (default: {DEFAULT_WINDOW_HOURS}).",
+    )
+    parser.add_argument(
+        "--allow-missing",
+        action="store_true",
+        help="Treat 'sink missing' as success — useful for first-run / CI smoke.",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Emit the HealthReport as JSON instead of plain text.",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    report = evaluate(
+        consumer_root=args.consumer_root,
+        window_hours=args.window_hours,
+    )
+
+    if args.json:
+        print(json.dumps(report.as_dict(), separators=(",", ":")))
+    else:
+        icon = {"healthy": "✅", "silent": "❌", "missing": "⚠️", "unreadable": "❌"}[report.status]
+        print(f"{icon}  {report.message}")
+        if report.last_ts:
+            print(f"   last record: {report.last_ts}")
+        print(f"   sink: {report.path}")
+
+    if report.status == "healthy":
+        return 0
+    if report.status == "missing" and args.allow_missing:
+        return 0
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/mcp_telemetry_query.py

@@ -0,0 +1,203 @@
+"""MCP telemetry query CLI — Phase 2 K2.
+
+Reads the SQLite store written by ``scripts/mcp_telemetry_store.py``
+and surfaces:
+
+- Per-tool attempt counts.
+- Distinct-consumer counts (``client_id_hash``).
+- Outcome ratios (``implemented`` / ``stub`` / ``latent_demand``).
+- Latent-demand names — tool names not in the catalog.
+
+Refresh cadence: cheap enough to run on every ``task mcp:report``
+invocation. Stdlib-only; reads (never writes) the DB.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from mcp_telemetry_store import resolve_db  # noqa: E402
+
+# Lazy import the catalog so the query CLI stays usable when only the
+# JSONL is present (e.g. an analyst node without the full package).
+try:
+    from mcp_server.catalog import load_catalog  # noqa: E402
+
+    _CATALOG_AVAILABLE = True
+except Exception:  # pragma: no cover - defensive
+    _CATALOG_AVAILABLE = False
+
+
+@dataclass(frozen=True)
+class ToolRow:
+    tool_name: str
+    attempts: int
+    distinct_consumers: int
+    implemented: int
+    stub: int
+    latent_demand: int
+    last_ts: str | None
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "tool_name": self.tool_name,
+            "attempts": self.attempts,
+            "distinct_consumers": self.distinct_consumers,
+            "implemented": self.implemented,
+            "stub": self.stub,
+            "latent_demand": self.latent_demand,
+            "last_ts": self.last_ts,
+        }
+
+
+def _connect_ro(db_path: Path) -> sqlite3.Connection:
+    if not db_path.exists():
+        raise FileNotFoundError(f"telemetry db not found: {db_path}")
+    uri = f"file:{db_path.as_posix()}?mode=ro"
+    return sqlite3.connect(uri, uri=True)
+
+
+def _query_tools(conn: sqlite3.Connection) -> list[ToolRow]:
+    rows = conn.execute(
+        """
+        SELECT
+          tool_name,
+          COUNT(*) AS attempts,
+          COUNT(DISTINCT client_id_hash) AS distinct_consumers,
+          SUM(CASE WHEN outcome = 'implemented' THEN 1 ELSE 0 END),
+          SUM(CASE WHEN outcome = 'stub' THEN 1 ELSE 0 END),
+          SUM(CASE WHEN outcome = 'latent_demand' THEN 1 ELSE 0 END),
+          MAX(ts)
+        FROM calls
+        GROUP BY tool_name
+        ORDER BY attempts DESC, tool_name ASC
+        """
+    ).fetchall()
+    return [
+        ToolRow(
+            tool_name=r[0],
+            attempts=r[1],
+            distinct_consumers=r[2],
+            implemented=r[3] or 0,
+            stub=r[4] or 0,
+            latent_demand=r[5] or 0,
+            last_ts=r[6],
+        )
+        for r in rows
+    ]
+
+
+def _catalog_names() -> frozenset[str]:
+    if not _CATALOG_AVAILABLE:
+        return frozenset()
+    try:
+        return frozenset(e.name for e in load_catalog())
+    except Exception:
+        return frozenset()
+
+
+def summarise(db_path: Path) -> dict[str, Any]:
+    """Pure-function summary used by both the CLI and the tests."""
+    conn = _connect_ro(db_path)
+    try:
+        tools = _query_tools(conn)
+    finally:
+        conn.close()
+    catalog = _catalog_names()
+    total_attempts = sum(t.attempts for t in tools)
+    total_consumers = len({})  # placeholder; recomputed below
+    consumer_set: set[str] = set()
+    conn2 = _connect_ro(db_path)
+    try:
+        for (cid,) in conn2.execute(
+            "SELECT DISTINCT client_id_hash FROM calls"
+        ):
+            consumer_set.add(cid)
+    finally:
+        conn2.close()
+    total_consumers = len(consumer_set)
+    latent_names = [
+        t.tool_name for t in tools
+        if t.latent_demand > 0
+        and catalog
+        and t.tool_name not in catalog
+    ]
+    return {
+        "db_path": str(db_path),
+        "total_attempts": total_attempts,
+        "total_distinct_consumers": total_consumers,
+        "tools": [t.as_dict() for t in tools],
+        "latent_demand_names": sorted(set(latent_names)),
+        "catalog_known": len(catalog) > 0,
+    }
+
+
+def _print_human(report: dict[str, Any]) -> None:
+    print(
+        f"📊  {report['total_attempts']} attempts across "
+        f"{len(report['tools'])} tool(s) — "
+        f"{report['total_distinct_consumers']} distinct consumer(s)"
+    )
+    print(f"   db: {report['db_path']}")
+    print()
+    if not report["tools"]:
+        print("(no telemetry rows — run scripts/mcp_telemetry_store.py first)")
+        return
+    print(
+        f"  {'tool':<28}  {'att':>5}  {'cons':>5}  "
+        f"{'impl':>5}  {'stub':>5}  {'lat':>5}  last_ts"
+    )
+    for t in report["tools"]:
+        print(
+            f"  {t['tool_name']:<28}  {t['attempts']:>5}  "
+            f"{t['distinct_consumers']:>5}  {t['implemented']:>5}  "
+            f"{t['stub']:>5}  {t['latent_demand']:>5}  "
+            f"{t['last_ts'] or '—'}"
+        )
+    if report["latent_demand_names"]:
+        print()
+        print("⚠️  latent-demand names not in catalog:")
+        for n in report["latent_demand_names"]:
+            print(f"   - {n}")
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Query the MCP telemetry SQLite store (Phase 2 K2)."
+    )
+    parser.add_argument("--consumer-root", type=Path, default=None)
+    parser.add_argument("--db", type=Path, default=None)
+    parser.add_argument("--json", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    db_path = args.db or resolve_db(args.consumer_root)
+    try:
+        report = summarise(db_path)
+    except FileNotFoundError as exc:
+        msg = (
+            f"❌  {exc}\n"
+            "   run `python3 scripts/mcp_telemetry_store.py` first."
+        )
+        if args.json:
+            print(json.dumps({"error": str(exc)}, separators=(",", ":")))
+        else:
+            print(msg, file=sys.stderr)
+        return 1
+    if args.json:
+        print(json.dumps(report, separators=(",", ":")))
+    else:
+        _print_human(report)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/mcp_telemetry_store.py

@@ -0,0 +1,211 @@
+"""MCP telemetry SQLite store — Phase 2 K1.
+
+Ingests the JSONL sink written by ``scripts/mcp_server/telemetry.py``
+into a queryable SQLite database. Idempotent: each JSONL line is hashed
+and stored as the primary key, so re-running ``ingest`` is safe and
+won't double-count records.
+
+Contract:
+
+- Source of truth stays the JSONL file. SQLite is a derived view.
+- Schema is documented in ``docs/contracts/mcp-telemetry-store.md``.
+- Stdlib-only — no SQLAlchemy / pandas, so consumers can run this
+  without extra dependencies.
+
+Phase 2 K2 (``scripts/mcp_telemetry_query.py``) reads from this store.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from mcp_server.telemetry import (  # noqa: E402
+    TELEMETRY_FILENAME,
+    TELEMETRY_REL_DIR,
+)
+
+DEFAULT_DB_REL = "agents/.mcp-telemetry/calls.sqlite3"
+SCHEMA_VERSION = 1
+
+
+@dataclass(frozen=True)
+class IngestReport:
+    """Outcome of one ``ingest`` run."""
+
+    source_path: str
+    db_path: str
+    lines_read: int
+    lines_skipped: int
+    rows_inserted: int
+    rows_already_present: int
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "source_path": self.source_path,
+            "db_path": self.db_path,
+            "lines_read": self.lines_read,
+            "lines_skipped": self.lines_skipped,
+            "rows_inserted": self.rows_inserted,
+            "rows_already_present": self.rows_already_present,
+        }
+
+
+def resolve_source(consumer_root: Path | None = None) -> Path:
+    root = (consumer_root or Path.cwd()).resolve()
+    return root / TELEMETRY_REL_DIR / TELEMETRY_FILENAME
+
+
+def resolve_db(consumer_root: Path | None = None) -> Path:
+    root = (consumer_root or Path.cwd()).resolve()
+    return root / DEFAULT_DB_REL
+
+
+def _connect(db_path: Path) -> sqlite3.Connection:
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(db_path)
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS calls (
+            line_hash TEXT PRIMARY KEY,
+            tool_name TEXT NOT NULL,
+            client_id_hash TEXT NOT NULL,
+            ts TEXT NOT NULL,
+            transport TEXT NOT NULL,
+            outcome TEXT NOT NULL,
+            ingested_at TEXT NOT NULL
+        )
+        """
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_calls_tool ON calls(tool_name)"
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_calls_ts ON calls(ts)"
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_calls_outcome ON calls(outcome)"
+    )
+    conn.commit()
+    return conn
+
+
+def _iter_lines(source: Path) -> Iterable[str]:
+    with source.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            yield line
+
+
+def ingest(
+    *,
+    consumer_root: Path | None = None,
+    source_override: Path | None = None,
+    db_override: Path | None = None,
+) -> IngestReport:
+    """Read the JSONL sink and upsert into SQLite. Idempotent."""
+    source = source_override or resolve_source(consumer_root)
+    db_path = db_override or resolve_db(consumer_root)
+
+    if not source.exists():
+        # Still create the DB so K2 has something to query against.
+        conn = _connect(db_path)
+        conn.close()
+        return IngestReport(
+            source_path=str(source),
+            db_path=str(db_path),
+            lines_read=0,
+            lines_skipped=0,
+            rows_inserted=0,
+            rows_already_present=0,
+        )
+
+    conn = _connect(db_path)
+    inserted = 0
+    already = 0
+    read = 0
+    skipped = 0
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    try:
+        for raw in _iter_lines(source):
+            stripped = raw.strip()
+            if not stripped:
+                continue
+            read += 1
+            try:
+                record = json.loads(stripped)
+            except ValueError:
+                skipped += 1
+                continue
+            line_hash = hashlib.sha256(stripped.encode("utf-8")).hexdigest()
+            cur = conn.execute(
+                "INSERT OR IGNORE INTO calls "
+                "(line_hash, tool_name, client_id_hash, ts, transport, "
+                "outcome, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (
+                    line_hash,
+                    str(record.get("tool_name", "")),
+                    str(record.get("client_id_hash", "")),
+                    str(record.get("ts", "")),
+                    str(record.get("transport", "")),
+                    str(record.get("outcome", "")),
+                    now,
+                ),
+            )
+            if cur.rowcount == 1:
+                inserted += 1
+            else:
+                already += 1
+        conn.commit()
+    finally:
+        conn.close()
+    return IngestReport(
+        source_path=str(source),
+        db_path=str(db_path),
+        lines_read=read,
+        lines_skipped=skipped,
+        rows_inserted=inserted,
+        rows_already_present=already,
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Ingest MCP telemetry JSONL into SQLite (Phase 2 K1)."
+    )
+    parser.add_argument("--consumer-root", type=Path, default=None)
+    parser.add_argument("--source", type=Path, default=None)
+    parser.add_argument("--db", type=Path, default=None)
+    parser.add_argument("--json", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    report = ingest(
+        consumer_root=args.consumer_root,
+        source_override=args.source,
+        db_override=args.db,
+    )
+    if args.json:
+        print(json.dumps(report.as_dict(), separators=(",", ":")))
+    else:
+        print(
+            f"✅  ingested {report.rows_inserted} new row(s) "
+            f"(skipped {report.lines_skipped} malformed, "
+            f"{report.rows_already_present} already present)"
+        )
+        print(f"   source: {report.source_path}")
+        print(f"   db:     {report.db_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/memory_signal.py

@@ -49,18 +49,20 @@ def _skip_intake_when_present() -> bool:
     Default: False — intake JSONL is always written as debug trail even
     when the `agent-memory` backend is present (see
     `road-to-memory-merge-safety.md` Phase 3).
+
+    Centralized loader (road-to-portable-dev-preferences P3): tolerance
+    contract handles missing file / malformed YAML / no PyYAML uniformly.
     """
-    if not SETTINGS_FILE.is_file():
-        return False
-    try:
-        import yaml  # type: ignore
-    except ImportError:
-        return False
     try:
-        data = yaml.safe_load(SETTINGS_FILE.read_text(encoding="utf-8")) or {}
-    except Exception:
-        return False
-    mem = data.get("memory") if isinstance(data, dict) else None
+        from scripts._lib.agent_settings import load_agent_settings
+    except ImportError:  # pragma: no cover — script-style invocation
+        import sys as _sys
+        from pathlib import Path as _Path
+        _sys.path.insert(0, str(_Path(__file__).resolve().parent))
+        from _lib.agent_settings import load_agent_settings  # type: ignore[import-not-found]
+
+    data = load_agent_settings(project_path=SETTINGS_FILE)
+    mem = data.get("memory")
     if not isinstance(mem, dict):
         return False
     intake = mem.get("intake")