agent-bios 0.4.0 → 0.7.0

package/scripts/migrate-learnings.py

@@ -0,0 +1,423 @@
+#!/usr/bin/env python3
+"""Promote -> migrate: clear the personal copy of promoted learnings (Phase 4).
+
+After a push lands, a learning that was promoted into the shared corpus is now
+loaded from the corpus — its personal copy (written by `learn!`,
+scripts/collect-learning.py) would double-load. This user-side tool removes the
+absorbed personal copy, and ONLY it (never the user's hand-written entry
+CLAUDE.md `## Personal` section).
+
+THE SAFETY RULE (design/corpus-domain-packaging.md:161-168): remove a personal
+copy ONLY after verifying the promoted item is actually in THIS user's assembled
+bundle. Per-domain opt-in means a promotion into a package the user did NOT
+install must NOT trigger removal — that would silently lose the learning. When
+unsure, KEEP (a kept duplicate is redundant; a wrong removal is data loss).
+
+Bundle membership (mirrors scripts/assemble.py `kept`): full install (original
+single-zone / no selection) -> everything installed; universal tier (core/infra)
+-> always installed; a domain key -> installed iff in the user's selection;
+anything else (env-personal / unclassified / unknown) -> KEEP.
+
+Run per host (mirrors collect-learning: --host + --config-dir); install.sh calls
+it after the corpus deploy. Manifest absent/empty -> no-op.
+"""
+import argparse
+import importlib.util
+import json
+import os
+import pathlib
+import re
+import shutil
+import sys
+import time
+
+REPO = pathlib.Path(__file__).resolve().parent.parent
+MANIFEST = REPO / "config" / "promotions.json"
+DOMAINS = REPO / "config" / "domains.json"
+UNIVERSAL_TIERS = frozenset({"core", "infra"})  # == assemble.audience UNIVERSAL
+# learning_id inside a personal bullet's TRAILING comment (collect-learning
+# prose_bullet: "... <!-- learning_id: <uuid> created: <ts> -->"). Anchored to
+# end-of-line so a lesson body that quotes a `<!-- learning_id: … -->` string
+# can never shadow the bullet's own id (this corpus is about agent tooling).
+BULLET_LID_RE = re.compile(r"<!--\s*learning_id:\s*([0-9a-fA-F-]+)[^>]*-->\s*$")
+
+
+def die(msg, code=1):
+    print(f"migrate-learnings: {msg}", file=sys.stderr)
+    sys.exit(code)
+
+
+def _load(name, filename):
+    path = REPO / "scripts" / filename
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def load_collect():
+    """Reuse collect-learning.py's host/home/marker constants (single source)."""
+    return _load("collect_learning", "collect-learning.py")
+
+
+def load_manifest(path=MANIFEST):
+    if not path.is_file():
+        return []
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return [p for p in data.get("promotions", [])
+            if isinstance(p, dict) and isinstance(p.get("learning_id"), str)]
+
+
+def load_domain_keys(path=DOMAINS):
+    return set(json.loads(path.read_text(encoding="utf-8")).get("domains", {}))
+
+
+def make_in_bundle(full, selection, domain_keys):
+    """A promotion's placement domain -> is it in THIS user's bundle? Biased to
+    KEEP on anything not provably installed."""
+    selection = set(selection or ())
+
+    def in_bundle(domain):
+        if full:
+            return True
+        if domain in UNIVERSAL_TIERS:
+            return True
+        if domain in domain_keys:
+            return domain in selection
+        return False  # env-personal / unclassified / unknown -> keep
+
+    return in_bundle
+
+
+def backup(path):
+    shutil.copy2(path, path.with_suffix(path.suffix + f".bak-migrate-{time.strftime('%Y%m%d-%H%M%S')}"))
+
+
+def atomic_write(path, text):
+    """Back up, then replace via a temp file + os.replace so a crash mid-write
+    can never truncate the durable personal file (reuses collect-learning's
+    write discipline)."""
+    backup(path)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(text, encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def prune_jsonl(jsonl, remove_ids, dry):
+    """Drop records whose learning_id is being removed. Returns count removed."""
+    if not jsonl.is_file():
+        return 0
+    kept, removed = [], 0
+    for line in jsonl.read_text(encoding="utf-8").splitlines():
+        s = line.strip()
+        if s:
+            try:
+                rec = json.loads(s)
+                if isinstance(rec, dict) and rec.get("learning_id") in remove_ids:
+                    removed += 1
+                    continue
+            except json.JSONDecodeError:
+                pass  # keep malformed lines untouched
+        kept.append(line)
+    if removed and not dry:
+        atomic_write(jsonl, "\n".join(kept) + ("\n" if kept else ""))
+    return removed
+
+
+def prune_bullets(text, remove_ids):
+    """Drop bullet lines whose comment learning_id is being removed. Returns
+    (new_text, count). Non-bullet lines (header/comments) are preserved."""
+    out, removed = [], 0
+    for line in text.splitlines():
+        m = BULLET_LID_RE.search(line)
+        if m and m.group(1) in remove_ids:
+            removed += 1
+            continue
+        out.append(line)
+    new = "\n".join(out) + ("\n" if text.endswith("\n") and out else "")
+    return new, removed
+
+
+def prune_claude_prose(home, remove_ids, dry):
+    md = home / "personal" / "learnings.md"
+    if not md.is_file():
+        return 0
+    new, removed = prune_bullets(md.read_text(encoding="utf-8"), remove_ids)
+    if removed and not dry:
+        atomic_write(md, new)
+    return removed
+
+
+def prune_codex_prose(home, remove_ids, collect, dry):
+    agents = home / "AGENTS.md"
+    if not agents.is_file():
+        return 0
+    body = agents.read_text(encoding="utf-8")
+    start, end = collect.PERSONAL_START, collect.PERSONAL_END
+    if start not in body:
+        return 0
+    pre, rest = body.split(start, 1)
+    if end not in rest:   # malformed (missing, or END before START) → touch nothing
+        return 0
+    region, post = rest.split(end, 1)
+    new_region, removed = prune_bullets(region, remove_ids)
+    if removed and not dry:
+        atomic_write(agents, f"{pre}{start}{new_region}{end}{post}")
+    return removed
+
+
+def claude_corpus_loaded(home, full):
+    """Is the shared corpus actually LOADED for this claude home? Removing a
+    personal copy while the corpus copy is not loaded would make the rule vanish
+    (Review F2). Non-packaged (`full`): the corpus IS the entry monolith the
+    deploy wrote -> loaded. Packaged: loads only via the entry's
+    `@central/bundle.md` import; a user-owned entry that lacks it (assemble exit
+    2) is NOT loading central -> keep the personal copy."""
+    if full:
+        return True
+    entry = home / "CLAUDE.md"
+    return entry.is_file() and collect_central_import() in entry.read_text(encoding="utf-8")
+
+
+def collect_central_import():
+    return "@central/bundle.md"
+
+
+def migrate(home, host, promotions, in_bundle, collect, dry=False, corpus_loaded=True):
+    """Remove personal copies of promoted+in-bundle learnings present locally.
+    corpus_loaded=False (the corpus is not actually loaded for this host) -> keep
+    everything (never orphan a personal copy)."""
+    if not corpus_loaded:
+        return {"removed": 0, "jsonl_removed": 0, "prose_removed": 0,
+                "kept_not_in_bundle": 0, "skipped": "corpus-not-loaded"}
+    jsonl = home / "personal" / "learnings.jsonl"
+    local_ids = set()
+    if jsonl.is_file():
+        for line in jsonl.read_text(encoding="utf-8").splitlines():
+            s = line.strip()
+            if not s:
+                continue
+            try:
+                rec = json.loads(s)
+            except json.JSONDecodeError:
+                continue
+            lid = rec.get("learning_id") if isinstance(rec, dict) else None
+            if isinstance(lid, str):
+                local_ids.add(lid)
+
+    remove_ids, kept_not_in_bundle = set(), 0
+    for p in promotions:
+        lid = p["learning_id"]
+        if lid not in local_ids:
+            continue  # not held locally (never captured here, or already migrated)
+        if in_bundle(p.get("domain")):
+            remove_ids.add(lid)
+        else:
+            kept_not_in_bundle += 1  # promoted but not in THIS user's bundle -> keep
+
+    # Prune PROSE first, jsonl second: the gate keys off ids still in the jsonl
+    # (line ~"if lid not in local_ids: continue"), so if a prose write fails, the
+    # id survives in jsonl and a re-run re-enters and converges. The reverse order
+    # would strand the prose bullet forever after a jsonl-only success (Review F4).
+    if host == "codex":
+        prose_removed = prune_codex_prose(home, remove_ids, collect, dry)
+    else:
+        prose_removed = prune_claude_prose(home, remove_ids, dry)
+    jsonl_removed = prune_jsonl(jsonl, remove_ids, dry)
+
+    return {"removed": len(remove_ids), "jsonl_removed": jsonl_removed,
+            "prose_removed": prose_removed, "kept_not_in_bundle": kept_not_in_bundle}
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Clear personal copies of promoted learnings.")
+    # Not argparse-required, so --self-test runs standalone; validated below.
+    ap.add_argument("--host", choices=("claude", "codex"))
+    ap.add_argument("--config-dir", default=None,
+                    help="config home (default: $CLAUDE_CONFIG_DIR / $CODEX_HOME by host)")
+    ap.add_argument("--full", action="store_true",
+                    help="non-packaged install: the full corpus is installed")
+    ap.add_argument("--selection-file",
+                    help="packaged install: selection.json ({\"domains\":[...]})")
+    ap.add_argument("--domains", help="packaged: comma-separated selection (overrides --selection-file)")
+    ap.add_argument("--manifest", default=None,
+                    help="promotion manifest (default: config/promotions.json)")
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--self-test", action="store_true")
+    args = ap.parse_args()
+
+    if args.self_test:
+        _self_test()
+        return
+
+    if not args.host:
+        die("--host claude|codex is required")
+    if not args.full and args.selection_file is None and args.domains is None:
+        die("pass --full (non-packaged) or --selection-file/--domains (packaged)")
+    if args.full and (args.selection_file is not None or args.domains is not None):
+        die("--full is mutually exclusive with --selection-file/--domains")
+
+    promotions = load_manifest(pathlib.Path(args.manifest) if args.manifest else MANIFEST)
+    if not promotions:
+        print("migrate-learnings: no promotions in the manifest — nothing to migrate")
+        return
+
+    collect = load_collect()
+    home = collect.resolve_home(args.host, args.config_dir)
+
+    # F2 gate: only prune where the shared corpus is actually loaded. Codex always
+    # loads AGENTS.md's central region; claude loads central only via the entry
+    # import (or inline in a full install).
+    corpus_loaded = True if args.host == "codex" else claude_corpus_loaded(home, args.full)
+
+    if args.full:
+        in_bundle = make_in_bundle(True, (), set())
+    else:
+        if args.domains is not None:
+            selection = [d for d in args.domains.split(",") if d]
+        else:
+            sel_path = pathlib.Path(args.selection_file)
+            if not sel_path.is_file():
+                die(f"selection file not found: {sel_path} (pass --full for a non-packaged install)")
+            selection = json.loads(sel_path.read_text(encoding="utf-8")).get("domains", [])
+        in_bundle = make_in_bundle(False, selection, load_domain_keys())
+
+    s = migrate(home, args.host, promotions, in_bundle, collect,
+                dry=args.dry_run, corpus_loaded=corpus_loaded)
+    tag = "[dry] " if args.dry_run else ""
+    if s.get("skipped"):
+        print(f"migrate-learnings: {tag}host={args.host} skipped ({s['skipped']}) — "
+              "kept every personal copy")
+        return
+    print(f"migrate-learnings: {tag}host={args.host} removed={s['removed']} "
+          f"(jsonl={s['jsonl_removed']} prose={s['prose_removed']}) "
+          f"kept_not_in_bundle={s['kept_not_in_bundle']}")
+
+
+def _self_test():
+    """Temp-home verification: the safety gate (keep a promotion not in the
+    user's bundle), removal of an in-bundle promotion from BOTH prose + jsonl,
+    idempotency, and that the user's own '## Personal' text is untouched. Uses
+    collect-learning's own writers so the seeded format is authoritative."""
+    import tempfile
+
+    collect = load_collect()
+
+    def rec(lid, domain):
+        return {"lesson": f"lesson for {domain} " + "x" * 10, "domain": domain,
+                "supporting_sessions": ["claude:abcd1234"],
+                "learning_id": lid, "schema_version": 1, "created": "2026-07-21T00:00:00Z"}
+
+    L = {"core": "0f8c1c2a-4d1e-4abc-9def-0000000000a1",
+         "bb": "0f8c1c2a-4d1e-4abc-9def-0000000000b2",
+         "off": "0f8c1c2a-4d1e-4abc-9def-0000000000c3"}
+    promos = [{"learning_id": L["core"], "domain": "core"},
+              {"learning_id": L["bb"], "domain": "builder-base"},
+              {"learning_id": L["off"], "domain": "office-work"}]
+
+    def seed_claude():
+        home = pathlib.Path(tempfile.mkdtemp(prefix="migrate-selftest-"))
+        (home / "personal").mkdir(parents=True)
+        jsonl = home / "personal" / "learnings.jsonl"
+        with open(jsonl, "w", encoding="utf-8") as f:
+            for k, dom in (("core", "core"), ("bb", "builder-base"), ("off", "office-work")):
+                r = rec(L[k], dom)
+                f.write(json.dumps(r, ensure_ascii=False) + "\n")
+                collect.apply_claude(home, collect.prose_bullet(r), dry=False)
+        # a user-owned '## Personal' line that migrate must never touch
+        entry = home / "CLAUDE.md"
+        entry.write_text(entry.read_text(encoding="utf-8") + "\n## Personal\n- my own rule\n", encoding="utf-8")
+        return home
+
+    def local_ids(home):
+        return {json.loads(l)["learning_id"]
+                for l in (home / "personal" / "learnings.jsonl").read_text(encoding="utf-8").splitlines() if l.strip()}
+
+    def md_has(home, lid):
+        return lid in (home / "personal" / "learnings.md").read_text(encoding="utf-8")
+
+    checks = []
+
+    # 1) Packaged, selection={builder-base}: core (universal) + builder-base
+    #    removed; office-work KEPT (the silent-loss guard).
+    home = seed_claude()
+    in_bundle = make_in_bundle(False, ["builder-base"], {"builder-base", "office-work", "llm-pipeline-dev"})
+    s = migrate(home, "claude", promos, in_bundle, collect)
+    ids = local_ids(home)
+    checks.append(("packaged: removed core+builder-base", s["removed"] == 2 and s["kept_not_in_bundle"] == 1))
+    checks.append(("packaged: office-work KEPT (not selected)", L["off"] in ids and md_has(home, L["off"])))
+    checks.append(("packaged: core+bb gone from jsonl", L["core"] not in ids and L["bb"] not in ids))
+    checks.append(("packaged: core+bb gone from prose", not md_has(home, L["core"]) and not md_has(home, L["bb"])))
+    checks.append(("user '## Personal' untouched", "my own rule" in (home / "CLAUDE.md").read_text(encoding="utf-8")))
+    # idempotent re-run removes nothing more
+    s2 = migrate(home, "claude", promos, in_bundle, collect)
+    checks.append(("idempotent re-run", s2["removed"] == 0))
+
+    # 2) Full (non-packaged) install: everything in bundle -> all removed.
+    home = seed_claude()
+    s = migrate(home, "claude", promos, make_in_bundle(True, (), set()), collect)
+    checks.append(("full install: all 3 removed", s["removed"] == 3 and not local_ids(home)))
+
+    # 3) dry-run changes nothing on disk.
+    home = seed_claude()
+    before = (home / "personal" / "learnings.jsonl").read_text(encoding="utf-8")
+    migrate(home, "claude", promos, make_in_bundle(True, (), set()), collect, dry=True)
+    checks.append(("dry-run writes nothing",
+                   (home / "personal" / "learnings.jsonl").read_text(encoding="utf-8") == before))
+
+    # 4) codex host: seed the AGENTS.md region, migrate prunes it there.
+    chome = pathlib.Path(tempfile.mkdtemp(prefix="migrate-codex-"))
+    (chome / "personal").mkdir(parents=True)
+    with open(chome / "personal" / "learnings.jsonl", "w", encoding="utf-8") as f:
+        r = rec(L["bb"], "builder-base")
+        f.write(json.dumps(r, ensure_ascii=False) + "\n")
+        collect.apply_codex(chome, collect.prose_bullet(r), dry=False)
+    s = migrate(chome, "codex", promos, make_in_bundle(True, (), set()), collect)
+    agents_txt = (chome / "AGENTS.md").read_text(encoding="utf-8")
+    checks.append(("codex: bullet removed from AGENTS.md region",
+                   s["prose_removed"] == 1 and L["bb"] not in agents_txt))
+
+    # 5) F2 gate: a packaged claude entry lacking @central/bundle.md means the
+    #    corpus is NOT loaded -> keep everything (never orphan a personal copy).
+    home = seed_claude()  # apply_claude writes an entry WITHOUT @central/bundle.md
+    checks.append(("F2: unwired packaged entry -> corpus not loaded",
+                   claude_corpus_loaded(home, full=False) is False))
+    s = migrate(home, "claude", promos, make_in_bundle(False, ["builder-base"], {"builder-base"}),
+                collect, corpus_loaded=False)
+    checks.append(("F2: corpus-not-loaded keeps ALL",
+                   s.get("skipped") == "corpus-not-loaded"
+                   and local_ids(home) == {L["core"], L["bb"], L["off"]}))
+    entry = home / "CLAUDE.md"
+    entry.write_text(entry.read_text(encoding="utf-8") + "\n@central/bundle.md\n", encoding="utf-8")
+    checks.append(("F2: wired entry -> corpus loaded",
+                   claude_corpus_loaded(home, full=False) is True))
+    checks.append(("F2: full install always loaded", claude_corpus_loaded(home, full=True) is True))
+
+    # 6) F5: the TRAILING comment's id wins; a learning_id quoted in the lesson
+    #    body must never shadow it (else a jsonl/prose desync).
+    twin = ("- [core] cf <!-- learning_id: aaaa0000-0000-0000-0000-000000000000 "
+            "created: x --> here  <!-- learning_id: 11112222-3333-4444-5555-666677778888 "
+            "created: y -->\n")
+    _, r_real = prune_bullets(twin, {"11112222-3333-4444-5555-666677778888"})
+    _, r_quoted = prune_bullets(twin, {"aaaa0000-0000-0000-0000-000000000000"})
+    checks.append(("F5: trailing id removed, quoted id ignored", r_real == 1 and r_quoted == 0))
+
+    # 7) malformed codex markers (END before START) -> no-op, no corruption.
+    bad = pathlib.Path(tempfile.mkdtemp(prefix="migrate-badcodex-"))
+    agents = bad / "AGENTS.md"
+    agents.write_text(f"x {collect.PERSONAL_END} y {collect.PERSONAL_START} z", encoding="utf-8")
+    before = agents.read_text(encoding="utf-8")
+    rem = prune_codex_prose(bad, {"anything"}, collect, dry=False)
+    checks.append(("malformed codex markers -> no-op",
+                   rem == 0 and agents.read_text(encoding="utf-8") == before))
+
+    failed = [n for n, ok in checks if not ok]
+    if failed:
+        for n in failed:
+            print(f"migrate-learnings --self-test: FAIL: {n}", file=sys.stderr)
+        sys.exit(1)
+    print(f"migrate-learnings --self-test: OK ({len(checks)} migrate checks)")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/redact.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Secret-redaction floor — the ONE canonical secret scrubber for the repo.
+
+Both flows that egress session-derived text redact through here, so the floor
+is single-sourced (concept economy) rather than duplicated per call site:
+  * heavy flow  — scripts/session-distill/digest.py (per-session digests);
+  * light flow  — scripts/collect-learning.py (`learn!` prose + upload + the
+    curator export downstream, since the server stores the payload verbatim).
+
+`design/corpus-domain-packaging.md` declares this floor as an inherited
+constraint ("the digest pipeline's secret-redaction carries over to harvest").
+The patterns are deliberately conservative: kill obvious secrets/identifiers in
+the `key: value` / `key=value` form and well-known token shapes, keep prose. A
+`token`/`secret`/`bearer` word standing alone (no `:`/`=` + value) is NOT
+touched, so lessons ABOUT tokens (e.g. "trust the X-Hook-Token header") survive.
+
+This module has no side effects on import and is safe to load by path.
+scripts/redact.py is shipped in the npm package (package.json `files`) because
+the shipped scripts/collect-learning.py imports it at runtime.
+"""
+import re
+import sys
+
+# Conservative: kill obvious secrets/identifiers, keep prose.
+SECRET_RE = [
+    re.compile(r'(?i)(api[_-]?key|secret|token|password|authorization|bearer)\s*[:=]\s*\S+'),
+    re.compile(r'sk-[A-Za-z0-9_-]{16,}'),
+    re.compile(r'gh[pousr]_[A-Za-z0-9]{20,}'),
+    re.compile(r'eyJ[A-Za-z0-9_-]{20,}\.'),                       # JWT-ish
+    re.compile(r'AKIA[0-9A-Z]{12,}'),                             # AWS key id
+    re.compile(r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'),  # email
+]
+PLACEHOLDER = '<REDACTED>'
+
+
+def redact(s):
+    """Return `s` with obvious secrets/identifiers replaced by <REDACTED>.
+    Falsy/non-str input is returned unchanged (callers may pass None)."""
+    if not s or not isinstance(s, str):
+        return s
+    for r in SECRET_RE:
+        s = r.sub(PLACEHOLDER, s)
+    return s
+
+
+def _self_test():
+    """Prove each pattern fires and that clean prose survives; exits non-zero on
+    any miss so the umbrella gate catches a regression in the floor."""
+    must_redact = [
+        ("api_key value", "my api_key=sk_live_ABCDEF0123456789 leaked", "sk_live_"),
+        # key[:=]value form. NB the "Authorization: Bearer <token>" space/scheme
+        # form is only partially caught here (up to the scheme word) unless the
+        # token itself matches a shape below (JWT/sk-/ghp_/…) — a known floor
+        # boundary carried over from the heavy flow, not tightened in Phase 3.
+        ("authorization value", "set authorization=topsecretvalue123 here", "topsecretvalue123"),
+        ("openai key", "used sk-abcdefghij0123456789 here", "sk-abcdefghij"),
+        ("github token", "token ghp_abcdefghij0123456789klmn committed", "ghp_"),
+        ("aws key id", "key AKIA0123456789ABCD in env", "AKIA0123456789"),
+        ("jwt", "cookie eyJhbGciOiJIUzI1NiIsInR5cCI6.rest here", "eyJ"),
+        ("email", "ping alice.dev@example.com about it", "alice.dev@example.com"),
+    ]
+    must_keep = [
+        # a lesson ABOUT tokens/secrets, no `key: value` → untouched.
+        "trust the X-Hook-Token header; the server derives identity from it",
+        "the secret sauce is verifying the real dispatch path",
+        "password rotation policy matters but state no value",
+    ]
+    checks = []
+    for name, src, needle in must_redact:
+        out = redact(src)
+        checks.append((f"redacts {name}", PLACEHOLDER in out and needle not in out))
+    for src in must_keep:
+        out = redact(src)
+        checks.append((f"keeps: {src[:32]}…", out == src))
+    checks.append(("None passthrough", redact(None) is None))
+    checks.append(("empty passthrough", redact("") == ""))
+
+    failed = [name for name, ok in checks if not ok]
+    if failed:
+        for name in failed:
+            print(f"redact --self-test: FAIL: {name}", file=sys.stderr)
+        sys.exit(1)
+    print(f"redact --self-test: OK ({len(checks)} redaction-floor checks)")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "--self-test":
+        _self_test()
+    else:
+        # Filter mode: redact stdin → stdout (handy for ad-hoc use).
+        sys.stdout.write(redact(sys.stdin.read()))