agent-bios 0.4.0 → 0.5.0

package/scripts/collect-learning.py

@@ -0,0 +1,557 @@
+#!/usr/bin/env python3
+"""Deterministic submit tool for the session learning flow (trigger `learn!`).
+
+Capability boundary (design/collection-loop/DESIGN.md Phase 1 step 4): the LLM
+supplies ONLY the semantic payload (lesson, domain, supporting_sessions, and the
+optional criteria / classification / proposed_domain / context) plus its session
+`--host`; THIS script owns every deterministic value and side effect:
+
+  * mints learning_id (a lowercase UUID) + created (ISO-8601) + schema_version;
+  * validates the full record against config/learning.schema.json — the single
+    validation source, reused from scripts/check-learning.py (no second schema);
+  * logs the JSON record to <home>/personal/learnings.jsonl — the durable,
+    append-only upload source that the Phase 2 watermark drain re-sends from;
+  * writes the lesson prose where THIS host loads it next session, and wires it:
+      - claude: appends to <home>/personal/learnings.md and ensures the
+        `@personal/learnings.md` import once in the entry CLAUDE.md;
+      - codex: appends into a preserved `agent-bios:personal-learnings` region
+        of <home>/AGENTS.md (Codex has no @import; AGENTS.md is always loaded).
+        The region lives OUTSIDE the central markers so scripts/assemble.py —
+        which only rewrites the central region — preserves it across re-assembly.
+
+It REFUSES any script-owned field in the payload (deterministic values are never
+hand-authored) and, on a schema/membership violation, REJECTS and exits 1 — it
+never patches the payload to make it pass (runtime enforces, does not reason).
+
+Input:  the semantic payload as one JSON object on stdin.
+Host:   --host claude|codex (default: the tool prefix of supporting_sessions[0]).
+Home:   --config-dir, else $CLAUDE_CONFIG_DIR / $CODEX_HOME by host, else ~/.claude / ~/.codex.
+After the local writes it best-effort uploads not-yet-delivered records to
+{ingest-url}/api/ingest/learnings via a watermark over learnings.jsonl (Phase 2
+transport; skipped when the dashboard hook is not installed, or with --no-upload).
+"""
+import argparse
+import datetime
+import importlib.util
+import json
+import os
+import pathlib
+import shutil
+import sys
+import time
+import urllib.error
+import urllib.request
+import uuid
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+OWNED_FIELDS = ("schema_version", "learning_id", "created")
+
+# host -> (config-home env var, default home dirname under $HOME)
+HOSTS = {
+    "claude": ("CLAUDE_CONFIG_DIR", ".claude"),
+    "codex": ("CODEX_HOME", ".codex"),
+}
+
+CLAUDE_IMPORT_LINE = "@personal/learnings.md"
+CLAUDE_CENTRAL_IMPORT = "@central/bundle.md"
+
+# Codex AGENTS.md markers. The central pair is owned by scripts/assemble.py
+# (kept in sync here); the personal-learnings pair is this tool's own region,
+# placed outside the central pair so re-assembly preserves it.
+CENTRAL_START = "<!-- agent-bios:central:start -->"
+CENTRAL_END = "<!-- agent-bios:central:end -->"
+PERSONAL_START = "<!-- agent-bios:personal-learnings:start -->"
+PERSONAL_END = "<!-- agent-bios:personal-learnings:end -->"
+
+CLAUDE_LEARNINGS_HEADER = """# Personal learnings
+
+<!-- Automation-owned: written by the session learning flow (`learn!`,
+     scripts/collect-learning.py). Do NOT hand-edit — promote→migrate clears
+     applied items by learning_id when the org redistributes them. Your own
+     personal rules belong in the entry CLAUDE.md '## Personal' section, never
+     here. This file is pulled into context by the entry file's
+     `@personal/learnings.md` import. -->
+"""
+
+CODEX_REGION_HEADER = """## Personal learnings
+<!-- Automation-owned: written by the session learning flow (`learn!`,
+     scripts/collect-learning.py). Codex loads this via AGENTS.md (no @import).
+     Do NOT hand-edit — promote→migrate clears applied items by learning_id.
+     Kept outside the agent-bios central markers so re-assembly preserves it. -->
+"""
+
+
+def load_checker():
+    """Reuse scripts/check-learning.py as the single validation source."""
+    path = REPO / "scripts" / "check-learning.py"
+    spec = importlib.util.spec_from_file_location("check_learning", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def die(msg, code=1):
+    print(f"collect-learning: {msg}", file=sys.stderr)
+    sys.exit(code)
+
+
+def read_payload():
+    raw = sys.stdin.read()
+    if not raw.strip():
+        die("no payload on stdin (expected one JSON object with the semantic fields)")
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError as e:
+        die(f"payload is not valid JSON: {e}")
+    if not isinstance(payload, dict):
+        die("payload must be a JSON object")
+    present_owned = [f for f in OWNED_FIELDS if f in payload]
+    if present_owned:
+        die(f"payload must not carry script-owned field(s) {present_owned} — "
+            "collect-learning mints schema_version/learning_id/created itself")
+    return payload
+
+
+def resolve_host(cli_host, payload):
+    if cli_host:
+        return cli_host
+    sessions = payload.get("supporting_sessions")
+    if isinstance(sessions, list) and sessions and isinstance(sessions[0], str) and ":" in sessions[0]:
+        tool = sessions[0].split(":", 1)[0]
+        if tool in HOSTS:
+            return tool
+    die("cannot determine host — pass --host claude|codex "
+        "(or a supporting_sessions entry prefixed 'claude:'/'codex:')")
+
+
+def resolve_home(host, config_dir):
+    if config_dir:
+        return pathlib.Path(config_dir)
+    env_var, default_name = HOSTS[host]
+    return pathlib.Path(os.environ.get(env_var) or pathlib.Path.home() / default_name)
+
+
+def build_record(payload):
+    record = dict(payload)
+    record["schema_version"] = 1
+    record["learning_id"] = str(uuid.uuid4())  # canonical lowercase
+    record["created"] = datetime.datetime.now().astimezone().replace(microsecond=0).isoformat()
+    return record
+
+
+def validate(record):
+    checker = load_checker()
+    validator = checker.build_validator()
+    domain_values = checker.valid_domain_values()
+    return checker.validate_record(record, validator, domain_values)
+
+
+def prose_bullet(record):
+    return (f"- [{record['domain']}] {record['lesson']}  "
+            f"<!-- learning_id: {record['learning_id']} created: {record['created']} -->")
+
+
+def backup(path):
+    shutil.copy2(path, path.with_suffix(path.suffix + f".bak-learn-{time.strftime('%Y%m%d-%H%M%S')}"))
+
+
+def append_record(home, record, dry):
+    jsonl = home / "personal" / "learnings.jsonl"
+    if dry:
+        print(f"  [dry] append JSON record to {jsonl}")
+        return jsonl
+    jsonl.parent.mkdir(parents=True, exist_ok=True)
+    with open(jsonl, "a", encoding="utf-8") as f:
+        f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    return jsonl
+
+
+# ---- claude: personal/learnings.md + @personal/learnings.md import ----------
+
+def apply_claude(home, bullet, dry):
+    md = home / "personal" / "learnings.md"
+    if dry:
+        if not md.exists():
+            print(f"  [dry] create {md}")
+        print(f"  [dry] append prose to {md}")
+    else:
+        md.parent.mkdir(parents=True, exist_ok=True)
+        if not md.exists():
+            md.write_text(CLAUDE_LEARNINGS_HEADER, encoding="utf-8")
+        with open(md, "a", encoding="utf-8") as f:
+            f.write(bullet + "\n")
+    import_state = ensure_claude_import(home, dry)
+    return md, f"entry import ({home / 'CLAUDE.md'}): {import_state}"
+
+
+def ensure_claude_import(home, dry):
+    entry = home / "CLAUDE.md"
+    if not entry.exists():
+        if dry:
+            print(f"  [dry] create entry {entry} with import line")
+            return "created"
+        home.mkdir(parents=True, exist_ok=True)
+        entry.write_text(f"# CLAUDE.md\n\n{CLAUDE_IMPORT_LINE}\n", encoding="utf-8")
+        return "created"
+    body = entry.read_text(encoding="utf-8")
+    if CLAUDE_IMPORT_LINE in body:
+        return "present"
+    if dry:
+        print(f"  [dry] insert '{CLAUDE_IMPORT_LINE}' into {entry}")
+        return "inserted"
+    lines = body.splitlines(keepends=True)
+    idx = next((i for i, ln in enumerate(lines) if CLAUDE_CENTRAL_IMPORT in ln), 0)
+    backup(entry)
+    lines.insert(idx + 1, CLAUDE_IMPORT_LINE + "\n")
+    entry.write_text("".join(lines), encoding="utf-8")
+    return "inserted"
+
+
+# ---- codex: personal-learnings region inside AGENTS.md ----------------------
+
+def apply_codex(home, bullet, dry):
+    agents = home / "AGENTS.md"
+    if agents.exists():
+        body = agents.read_text(encoding="utf-8")
+    else:
+        # Degenerate (corpus not installed): seed EMPTY central markers so a
+        # later assemble fills them and keeps our region in the preserved tail.
+        body = f"{CENTRAL_START}\n{CENTRAL_END}\n"
+    if PERSONAL_START in body and PERSONAL_END in body:
+        pre, rest = body.split(PERSONAL_START, 1)
+        region_body, post = rest.split(PERSONAL_END, 1)
+        new_body = region_body.rstrip("\n") + "\n" + bullet + "\n"
+        new = f"{pre}{PERSONAL_START}{new_body}{PERSONAL_END}{post}"
+    else:
+        region = f"\n{PERSONAL_START}\n{CODEX_REGION_HEADER}{bullet}\n{PERSONAL_END}\n"
+        new = body.rstrip("\n") + "\n" + region
+    if dry:
+        print(f"  [dry] write personal-learnings region in {agents}")
+        return agents, f"AGENTS.md region ({agents}): updated"
+    agents.parent.mkdir(parents=True, exist_ok=True)
+    if agents.exists():
+        backup(agents)
+    agents.write_text(new, encoding="utf-8")
+    return agents, f"AGENTS.md region ({agents}): updated"
+
+
+APPLY = {"claude": apply_claude, "codex": apply_codex}
+
+
+# ── Phase 2 transport: watermark upload over the durable learnings.jsonl ───────
+#
+# collect-learning runs on-demand (per `learn!`), so the upload piggybacks here:
+# after the local writes, best-effort POST any not-yet-settled records to
+# {ingest-url}/api/ingest/learnings and record which learning_ids are settled in
+# a small state file (the watermark). The durable learnings.jsonl is the single
+# source and is never capped, so nothing is lost across a multi-day server outage
+# — unsettled records simply retry on the next `learn!`. A 2xx (incl. a duplicate
+# re-send, which the server dedups by learning_id) settles a record; 400/413
+# settle it as permanently rejected; any other status (401/403/404/429/503/5xx/
+# network) is transient and stops the drain (no retry storm) to retry next time.
+# (design/collection-loop/PHASE2-ENDPOINT-DESIGN.md D2.5.)
+
+INGEST_PATH = "/api/ingest/learnings"
+STATE_NAME = ".learnings-upload-state.json"
+UPLOAD_LIMIT = 25          # max POSTs per invocation
+UPLOAD_BUDGET_S = 5.0      # total wall-clock budget for the whole drain
+UPLOAD_TIMEOUT_S = 3.0     # per-request socket timeout
+PERMANENT_STATUSES = frozenset({400, 413})  # never succeeds → settle as dead
+
+
+def transport_config(home):
+    """Read the dashboard hook's token + ingest base URL (READ-ONLY; the hook dir
+    is dashboard-owned — we never write it). Returns ((token, base), None) or
+    (None, reason) when the hook is not installed (Q-E: caller prints a notice)."""
+    hook_dir = home / "hooks"
+    token_file = hook_dir / "token"
+    if not token_file.is_file():
+        return None, "dashboard hook not installed (no token)"
+    try:
+        token = token_file.read_text(encoding="utf-8").strip()
+    except OSError as e:
+        return None, f"cannot read hook token ({e})"
+    if not token:
+        return None, "dashboard hook token is empty"
+    base = ""
+    # The installer writes `dashboard-url`; a migrated install may also have
+    # `ingest-url`. Prefer ingest-url, fall back to dashboard-url.
+    for name in ("ingest-url", "dashboard-url"):
+        f = hook_dir / name
+        if f.is_file():
+            candidate = f.read_text(encoding="utf-8").strip()
+            if candidate:
+                base = candidate
+                break
+    if not base:
+        return None, "dashboard hook ingest URL not found"
+    return (token, base.rstrip("/")), None
+
+
+def read_jsonl_records(jsonl):
+    """(learning_id, record) for each well-formed line; malformed lines are
+    skipped so a single poison line never wedges the drain."""
+    out = []
+    if not jsonl.is_file():
+        return out
+    with open(jsonl, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            lid = rec.get("learning_id") if isinstance(rec, dict) else None
+            if isinstance(lid, str) and lid:
+                out.append((lid, rec))
+    return out
+
+
+def load_state(home):
+    path = home / "personal" / STATE_NAME
+    if not path.is_file():
+        return {"uploaded": set(), "dead": set()}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return {"uploaded": set(data.get("uploaded", [])),
+                "dead": set(data.get("dead", []))}
+    except (json.JSONDecodeError, OSError):
+        return {"uploaded": set(), "dead": set()}
+
+
+def save_state(home, state, present_ids):
+    # Prune settled ids no longer in the durable log (e.g. migrated out) so the
+    # state stays bounded to the log size. Atomic replace.
+    path = home / "personal" / STATE_NAME
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = {"uploaded": sorted(state["uploaded"] & present_ids),
+            "dead": sorted(state["dead"] & present_ids)}
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(data, ensure_ascii=False) + "\n", encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def classify_status(status):
+    if status is None:
+        return "transient"           # network error / timeout
+    if 200 <= status < 300:
+        return "ok"
+    if status in PERMANENT_STATUSES:
+        return "permanent"
+    return "transient"               # 401/403/404/429/503/5xx → retry next time
+
+
+def http_post(base, token, record, timeout=UPLOAD_TIMEOUT_S):
+    """POST one record; return the HTTP status int, or None on a network error."""
+    body = json.dumps(record, ensure_ascii=False).encode("utf-8")
+    req = urllib.request.Request(base + INGEST_PATH, data=body, method="POST")
+    req.add_header("Content-Type", "application/json")
+    req.add_header("X-Hook-Token", token)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status
+    except urllib.error.HTTPError as e:
+        return e.code
+    except (urllib.error.URLError, TimeoutError, OSError):
+        return None
+
+
+def drain_uploads(home, post_fn=http_post, now=time.monotonic,
+                  limit=UPLOAD_LIMIT, budget_s=UPLOAD_BUDGET_S):
+    """Best-effort upload of not-yet-settled learnings. post_fn/now are injectable
+    for the self-test (no sockets). Returns a summary dict."""
+    config, reason = transport_config(home)
+    if config is None:
+        return {"status": "skipped", "reason": reason,
+                "uploaded": 0, "dead": 0, "pending": None}
+    token, base = config
+
+    records = read_jsonl_records(home / "personal" / "learnings.jsonl")
+    present_ids = {lid for lid, _ in records}
+    state = load_state(home)
+    settled = state["uploaded"] | state["dead"]
+    candidates = [(lid, rec) for lid, rec in records if lid not in settled]
+
+    deadline = now() + budget_s
+    uploaded = dead = sent = 0
+    stopped = None
+    for lid, rec in candidates:
+        if sent >= limit:
+            stopped = "limit"
+            break
+        if now() >= deadline:
+            stopped = "budget"
+            break
+        kind = classify_status(post_fn(base, token, rec))
+        sent += 1
+        if kind == "ok":
+            state["uploaded"].add(lid)
+            uploaded += 1
+        elif kind == "permanent":
+            state["dead"].add(lid)
+            dead += 1
+        else:  # transient — server likely down / systemic; stop, retry next time
+            stopped = "transient"
+            break
+
+    save_state(home, state, present_ids)
+    pending = sum(1 for lid, _ in records
+                  if lid not in (state["uploaded"] | state["dead"]))
+    return {"status": "ok", "reason": stopped, "uploaded": uploaded,
+            "dead": dead, "pending": pending}
+
+
+def print_upload_summary(s):
+    if s["status"] == "skipped":
+        print(f"  upload -> skipped ({s['reason']}); "
+              "retries on a later learn! once the hook is installed")
+        return
+    tail = f" (stopped: {s['reason']})" if s["reason"] else ""
+    print(f"  upload -> uploaded={s['uploaded']} dead={s['dead']} "
+          f"pending={s['pending']}{tail}")
+
+
+def _self_test():
+    """Socket-free verification of the drain: status classification, watermark
+    advance, transient-stop (no storm), permanent-drop, no-token skip, poison
+    line. Injects post_fn/now; exits non-zero on any failure."""
+    import tempfile
+
+    def make_home(records, with_token=True):
+        home = pathlib.Path(tempfile.mkdtemp(prefix="learn-selftest-"))
+        (home / "personal").mkdir(parents=True)
+        with open(home / "personal" / "learnings.jsonl", "w", encoding="utf-8") as f:
+            for r in records:
+                f.write(r if isinstance(r, str) else json.dumps(r))
+                f.write("\n")
+        if with_token:
+            (home / "hooks").mkdir(parents=True)
+            (home / "hooks" / "token").write_text("tok", encoding="utf-8")
+            (home / "hooks" / "dashboard-url").write_text(
+                "https://example.test/", encoding="utf-8")
+        return home
+
+    def rec(n):
+        return {"learning_id": f"0f8c1c2a-4d1e-4abc-9def-{n:012d}",
+                "schema_version": 1, "lesson": "x" * 12, "domain": "core",
+                "created": "2026-07-20T00:00:00Z",
+                "supporting_sessions": ["claude:abcd1234"]}
+
+    checks = []
+
+    # 1) no token → skipped, no state file written (watermark unadvanced).
+    h = make_home([rec(1)], with_token=False)
+    s = drain_uploads(h)
+    checks.append(("no-token skip", s["status"] == "skipped"
+                   and not (h / "personal" / STATE_NAME).is_file()))
+
+    # 2) all ok → all uploaded, pending 0; re-run uploads nothing new.
+    h = make_home([rec(1), rec(2), rec(3)])
+    s = drain_uploads(h, post_fn=lambda *a: 200)
+    calls = {"n": 0}
+    def count_post(*a):
+        calls["n"] += 1
+        return 200
+    s2 = drain_uploads(h, post_fn=count_post)
+    checks.append(("all-ok then idempotent re-run",
+                   s["uploaded"] == 3 and s["pending"] == 0
+                   and calls["n"] == 0 and s2["uploaded"] == 0))
+
+    # 3) transient (500) → nothing settled, drain stops, pending == N; a later
+    #    200 run delivers everything (multi-day-outage recovery).
+    h = make_home([rec(1), rec(2)])
+    s = drain_uploads(h, post_fn=lambda *a: 500)
+    s2 = drain_uploads(h, post_fn=lambda *a: 200)
+    checks.append(("transient stop then recover",
+                   s["uploaded"] == 0 and s["pending"] == 2
+                   and s["reason"] == "transient"
+                   and s2["uploaded"] == 2 and s2["pending"] == 0))
+
+    # 4) permanent (400) → settled as dead, not retried.
+    h = make_home([rec(1)])
+    s = drain_uploads(h, post_fn=lambda *a: 400)
+    hits = {"n": 0}
+    def once(*a):
+        hits["n"] += 1
+        return 400
+    s2 = drain_uploads(h, post_fn=once)
+    checks.append(("permanent drop, not retried",
+                   s["dead"] == 1 and s["pending"] == 0 and hits["n"] == 0))
+
+    # 5) network error (None) is transient.
+    h = make_home([rec(1)])
+    s = drain_uploads(h, post_fn=lambda *a: None)
+    checks.append(("network error is transient",
+                   s["uploaded"] == 0 and s["pending"] == 1))
+
+    # 6) poison line skipped, valid records still delivered.
+    h = make_home(["{ not json", rec(1), ""])
+    s = drain_uploads(h, post_fn=lambda *a: 200)
+    checks.append(("poison line skipped", s["uploaded"] == 1 and s["pending"] == 0))
+
+    # 7) wall-clock budget stops the drain (fake clock jumps past the deadline).
+    clock = {"t": 0.0}
+    def fake_now():
+        clock["t"] += 10.0
+        return clock["t"]
+    h = make_home([rec(1), rec(2)])
+    s = drain_uploads(h, post_fn=lambda *a: 200, now=fake_now, budget_s=5.0)
+    checks.append(("budget stop", s["reason"] == "budget"))
+
+    failed = [name for name, ok in checks if not ok]
+    if failed:
+        for name in failed:
+            print(f"collect-learning --self-test: FAIL: {name}", file=sys.stderr)
+        sys.exit(1)
+    print(f"collect-learning --self-test: OK ({len(checks)} upload-drain checks)")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Submit a session learning (learn!).")
+    ap.add_argument("--host", choices=sorted(HOSTS),
+                    help="session host (default: tool prefix of supporting_sessions[0])")
+    ap.add_argument("--config-dir", default=None,
+                    help="config home (default: $CLAUDE_CONFIG_DIR / $CODEX_HOME by host)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="validate and print actions; write nothing")
+    ap.add_argument("--no-upload", action="store_true",
+                    help="skip the Phase 2 upload (local writes only)")
+    ap.add_argument("--self-test", action="store_true",
+                    help="run the upload-drain self-test and exit")
+    args = ap.parse_args()
+
+    if args.self_test:
+        _self_test()
+        return
+
+    payload = read_payload()
+    host = resolve_host(args.host, payload)
+    home = resolve_home(host, args.config_dir)
+    record = build_record(payload)
+
+    errors = validate(record)
+    if errors:
+        print("collect-learning: REJECTED (record is not valid; not written)", file=sys.stderr)
+        for e in errors:
+            print(f"  - {e}", file=sys.stderr)
+        sys.exit(1)
+
+    dry = args.dry_run
+    bullet = prose_bullet(record)
+    jsonl = append_record(home, record, dry)
+    prose_path, wiring = APPLY[host](home, bullet, dry)
+
+    print(f"collect-learning: {'[dry] ' if dry else ''}OK  host={host} "
+          f"learning_id={record['learning_id']} domain={record['domain']}")
+    print(f"  prose  -> {prose_path}")
+    print(f"  record -> {jsonl}")
+    print(f"  {wiring}")
+
+    if not dry and not args.no_upload:
+        print_upload_summary(drain_uploads(home))
+
+
+if __name__ == "__main__":
+    main()