@r1z4x/tezgah 1.3.0 → 1.4.0

package/hooks/tezgah_lessons.py

@@ -7,16 +7,16 @@ The session block, the per-turn block and the per-turn digest
 ledger here, so they agree on what a lesson is and on its key. A module of its
 own because the gate must not import `tezgah_context` (its import cost, and the
 rule `hooks/tezgah_task.py::repo_root` states): this one imports only
-`tezgah_paths` at load, and `tezgah_gate` lazily, only for a line that names a
-gate rule as its enforcer.
+`tezgah_paths` and `tezgah_store` at load, and `tezgah_gate` lazily, only for a
+line that names a gate rule as its enforcer.
 
 Stdlib only."""
 import hashlib
-import json
 import os
 import re
 
-from tezgah_paths import cache_dir, off
+import tezgah_store
+from tezgah_paths import off
 
 # A line carries its metadata after its text, never in it: from the first
 # `|| check:` or `|| enforced_by:` clause outside a code span to the end of the
@@ -114,73 +114,28 @@ def is_ledger(path):
             and os.path.basename(os.path.dirname(path)) == ".tezgah")
 
 
-def taint_path(root):
-    """The `lesson_tainted` index of the repository at `root`: the gate keeps
-    each keyed row here as well as in the session's ledger, so the context of a
-    later session reads one small file for the per-line label instead of every
-    ledger on the machine (3173 files, 30 MB on the measuring machine,
-    2026-10-07)."""
-    name = hashlib.sha1(os.path.realpath(root).encode("utf-8", "replace"))
-    return os.path.join(cache_dir(), "lessons", name.hexdigest()[:16] + ".jsonl")
-
-
-def _index_rows(path):
-    """The parseable keyed rows of an index file, oldest first; [] without one.
-    A damaged line costs only itself."""
-    rows = []
-    try:
-        with open(path, encoding="utf-8", errors="replace") as fh:
-            for ln in fh:
-                try:
-                    row = json.loads(ln)
-                except ValueError:
-                    continue
-                if isinstance(row, dict) and row.get("key"):
-                    rows.append(row)
-    except OSError:
-        pass
-    return rows
+def taint_root(root):
+    """The key of the repository at `root` in the `lesson_taint` store: the gate
+    keeps each keyed row there as well as in the session's ledger, so the
+    context of a later session reads one repository's rows for the per-line
+    label instead of every ledger on the machine (3173 files, 30 MB on the
+    measuring machine, 2026-10-07)."""
+    return hashlib.sha1(os.path.realpath(root).encode("utf-8", "replace")).hexdigest()[:16]
 
 
 def write_taint(root, keep, rows):
     """Rewrite the index of the repository at `root`: the rows it holds whose key
-    is in `keep` (the ledger's lines plus the write's new ones), then `rows`. So
-    the file never outgrows the ledger it labels. Best effort, like a ledger row.
-    ponytail: two sessions rewriting one index in the same instant can lose one
-    row (one label); the session ledger keeps its own copy of every row."""
-    path = taint_path(root)
-    kept = [r for r in _index_rows(path) if r["key"] in keep] + rows
-    tmp = "%s.%d.tmp" % (path, os.getpid())
-    try:
-        os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
-        with os.fdopen(fd, "w", encoding="utf-8") as fh:
-            fh.write("".join(json.dumps(r) + "\n" for r in kept))
-        os.replace(tmp, path)
-    except OSError:
-        pass
-
-
-# The index as last read, per (path, mtime, size): the session block, the
-# per-turn block and the digest of one prompt each ask for it, and a hook
-# process serves one prompt.
-_TAINTED = {}
+    is in `keep` (the ledger's lines plus the write's new ones), then `rows`, in
+    one transaction. So the index never outgrows the ledger it labels. Best
+    effort, like a ledger row."""
+    tezgah_store.write_taint(taint_root(root), keep, rows)
 
 
 def tainted(root):
     """{key: source} for the lessons of the repository at `root` that a gate
-    recorded as written in a turn that had read untrusted text (`taint_path`);
-    {} when there is none. Read once while the file is unchanged."""
-    path = taint_path(root)
-    try:
-        st = os.stat(path)
-    except OSError:
-        return {}
-    key = (path, st.st_mtime_ns, st.st_size)
-    if key not in _TAINTED:
-        _TAINTED[key] = {str(r["key"]): str(r.get("source") or "")
-                         for r in _index_rows(path)}
-    return _TAINTED[key]
+    recorded as written in a turn that had read untrusted text (`taint_root`);
+    {} when there is none."""
+    return tezgah_store.taint(taint_root(root))
 
 
 def lines(root, retired=None):

package/hooks/tezgah_paths.py

@@ -612,50 +612,40 @@ def reply_lang():
 # `authorized` row, the stand-down text) - the control rule refuses the routes
 # the gate sees, and an interpreter writing them is SECURITY.md's residual.
 STAND_DOWN = "tezgah-setup --uninstall --full\n"
-_LATCH = {"ledger": None, "since": None, "auth": None}
+_LATCH = {"ledger": None, "since": None}
 
 
 def latch(ledger):
     """Bind this process's `off()` to one session's ledger path (None unbinds)."""
-    _LATCH.update(ledger=ledger, since=None, auth=None)
+    _LATCH.update(ledger=ledger, since=None)
 
 
 def _first_ts(ledger):
+    import tezgah_store  # deferred: an unbound process never opens the ledger
     try:
-        with open(ledger, "rb") as fh:
-            ts = json.loads(fh.readline()).get("ts")
-    except (OSError, ValueError, AttributeError):
+        ts = json.loads(tezgah_store.evidence_first(ledger) or "null").get("ts")
+    except (*tezgah_store.ERRORS, ValueError, AttributeError):
         return None
     return ts if isinstance(ts, (int, float)) else None
 
 
 def _authorized(ledger):
-    """The switch names this session's `authorized` rows carry, memoized on
-    the ledger's size: read only when a switch is newer than the first row."""
-    try:
-        size = os.path.getsize(ledger)
-    except OSError:
-        return frozenset()
-    memo = _LATCH["auth"]
-    if memo and memo[0] == size:
-        return memo[1]
+    """The switch names this session's `authorized` rows carry: read only when
+    a switch is newer than the first row."""
+    import tezgah_store
     names = set()
     try:
-        with open(ledger, encoding="utf-8", errors="replace") as fh:
-            for line in fh:
-                if '"authorized"' not in line:
-                    continue
-                try:
-                    row = json.loads(line)
-                except ValueError:
-                    continue
-                if row.get("kind") == "authorized" and isinstance(
-                        row.get("authorized"), list):
-                    names.update(n for n in row["authorized"] if isinstance(n, str))
-    except OSError:
+        found = tezgah_store.evidence_rows(ledger, kind="authorized")
+    except tezgah_store.ERRORS:
         return frozenset()
-    _LATCH["auth"] = (size, frozenset(names))
-    return _LATCH["auth"][1]
+    for text in found:
+        try:
+            row = json.loads(text)
+        except ValueError:
+            continue
+        if isinstance(row, dict) and isinstance(row.get("authorized"), list):
+            names.update(n for n in row["authorized"] if isinstance(n, str))
+    return frozenset(names)
 
 
 def _honored(path, name):

package/hooks/tezgah_replay.py

@@ -40,8 +40,10 @@ call's own output.
 
 **Output.** Everything is written under `~/.cache/tezgah/replay/<run>/`, owner
 only, and never leaves the machine: `corpus.jsonl` (the items, with their
-inputs), `results.jsonl`, `summary.json`, and the sheet files. The sandbox HOME
-and the row stream are deleted when the child returns.
+inputs), `results.jsonl` and the sheet files. The run's summary is its row in
+the cache database's replay index (`tezgah_store.add_replay`), whose newest
+row is the latest run. The sandbox HOME and the row stream are deleted when
+the child returns.
 """
 import argparse
 import datetime
@@ -64,6 +66,7 @@ if HERE not in sys.path:
     sys.path.insert(0, HERE)
 import tezgah_integrity as ti  # noqa: E402
 import tezgah_paths as tp  # noqa: E402
+import tezgah_store  # noqa: E402
 
 # The one rule-label derivation; nothing in this module maps text to a rule.
 rule_of = ti._deny_rule
@@ -425,9 +428,6 @@ def child(run):
     import tezgah_gate as tg
     items = {it["i"]: it for it in _read_jsonl(os.path.join(run, "corpus.jsonl"))}
     evidence = os.path.join(ti.cache_dir(), "evidence")
-    nudged = os.path.join(ti.cache_dir(), "nudged")
-    os.makedirs(evidence, mode=0o700, exist_ok=True)
-    os.makedirs(nudged, mode=0o700, exist_ok=True)
     now = [0.0]
 
     def frozen():
@@ -454,15 +454,13 @@ def child(run):
                 now[0] = float(it["ts"])
                 results.append(_replay_one(it, rows_of[fname], tg, seen, live_events))
             rows_of[fname].append(row)
-            path = os.path.join(evidence, fname)
-            with open(path, "a", encoding="utf-8") as out:
-                out.write(json.dumps(row) + "\n")
-            if isinstance(row.get("ts"), (int, float)):
-                os.utime(path, (row["ts"], row["ts"]))
+            # the row as the live writer stored it; its own `ts` is the time the
+            # cross-session read (`writers_elsewhere`) windows on
+            ti._append(os.path.join(evidence, fname), json.dumps(row) + "\n")
             if row.get("kind") == "nudge":
                 # the once-per-session nudge mark the live gate wrote with this
-                # row (`first_nudge`): disk state the ledger records
-                open(tg.nudge_mark(sid), "w", encoding="utf-8").close()
+                # row (`first_nudge`): cache state the ledger records
+                tezgah_store.mark("nudged", tg.nudge_key(sid))
     _write_jsonl(os.path.join(run, "results.jsonl"), results)
 
 
@@ -597,8 +595,7 @@ def run_replay(cutoff=None, since=None):
                "fidelity": fidelity(items, results),
                "race_family": family,
                "deny_runs": folds}
-    _owner_write(os.path.join(run, "summary.json"), json.dumps(summary, indent=2) + "\n")
-    _owner_write(os.path.join(replay_root(), "latest"), run + "\n")
+    tezgah_store.add_replay(_run_key(run), summary, tp.CACHE)
     return summary
 
 
@@ -671,7 +668,7 @@ def sheet(run, seed=SEED, rules=None):
     """The blind sample: `sheet.jsonl` ({set, n, text}), `sheet-key.jsonl` (the
     hidden live verdicts) and `instructions.md`. Returns (paths, counts)."""
     items = _read_jsonl(os.path.join(run, "corpus.jsonl"))
-    summary = json.load(open(os.path.join(run, "summary.json"), encoding="utf-8"))
+    summary = _summary(run)
     rng = random.Random(seed)
     pools = defaultdict(list)
     # A race deny written before the ledger stored `target` cannot show its
@@ -790,7 +787,7 @@ def _rate(k, n):
 
 
 def report(run, label_files=()):
-    summary = json.load(open(os.path.join(run, "summary.json"), encoding="utf-8"))
+    summary = _summary(run)
     out = {k: summary[k] for k in ("run", "cutoff", "cutoff_local", "ledgers", "candidates",
                                    "excluded", "items", "joined", "join",
                                    "stop_text_fidelity", "fidelity", "race_family",
@@ -1010,8 +1007,24 @@ def _pct(k, n):
 def _run_dir(arg):
     if arg:
         return arg
-    with open(os.path.join(replay_root(), "latest"), encoding="utf-8") as fh:
-        return fh.read().strip()
+    run = tezgah_store.replay_latest(tp.CACHE)
+    if not run:
+        raise SystemExit("replay: no run yet; run `tezgah-gate replay` first")
+    return run
+
+
+def _run_key(run):
+    """A run's key in the replay index: its directory's real path, so a
+    relative or linked `--run` names the run its row was written under."""
+    return os.path.realpath(run)
+
+
+def _summary(run):
+    """The summary the run in directory `run` was indexed with (`run_replay`)."""
+    summary = tezgah_store.replay_summary(_run_key(run), tp.CACHE)
+    if summary is None:
+        raise SystemExit("replay: no indexed run at %s" % run)
+    return summary
 
 
 def main(argv):

package/hooks/tezgah_shapes.py

@@ -212,7 +212,7 @@ def rule_yield(files=None, shipped=None):
     `rule_ship_times()`. Fixture ledgers are left out by `ti.fixture_ledger`, the
     exclusion `counters_all` applies, because a probe run measures the harness
     and not use. A rule's exposure is the real ledgers last written at or after
-    it shipped (the file's mtime), and its fires are the deny rows under its
+    it shipped (the newest row's `ts`), and its fires are the deny rows under its
     label in those ledgers. `mark` is `retire-candidate` at or below
     `RETIRE_FIRES` fires over at least `MIN_EXPOSURE` ledgers, `low-exposure` at
     or below `RETIRE_FIRES` over fewer (or an unknown ship date), else None.
@@ -227,9 +227,9 @@ def rule_yield(files=None, shipped=None):
         rows = ti._foreign_rows(path)
         if ti.fixture_ledger(rows):
             continue
-        try:
-            when = os.path.getmtime(path)
-        except OSError:
+        when = max((r["ts"] for r in rows if isinstance(r.get("ts"), (int, float))),
+                   default=None)
+        if when is None:
             continue
         real.append((when, Counter(_label(row.get("detail")) for row in rows
                                    if row.get("kind") == "deny")))

package/hooks/tezgah_skill_pick.py

@@ -58,6 +58,7 @@ import tezgah_integrity as ti
 import tezgah_judge
 import tezgah_rank
 import tezgah_paths as tp
+import tezgah_store
 
 ARM = "skill-suggest-on"
 # Below this, the turn's own gate question says no skill is wanted and nothing is
@@ -216,17 +217,8 @@ def judge(prompt, names, session_id=""):
                             roots=[os.path.join(tp.PLUGIN_ROOT, "skills")]))
 
 
-def _path(session_id):
-    return os.path.join(tp.cache_dir(), "skill-pick",
-                        slug(session_id) + ".json")
-
-
 def _remembered(session_id):
-    try:
-        with open(_path(session_id), encoding="utf-8") as fh:
-            data = json.load(fh)
-    except (OSError, ValueError):
-        return {}
+    data = tezgah_store.doc("skill_pick", slug(session_id))
     return data if isinstance(data, dict) else {}
 
 
@@ -237,13 +229,7 @@ def _remember(session_id, digest, line):
     data[digest] = line
     for old in list(data)[:max(0, len(data) - KEPT)]:
         del data[old]
-    path = _path(session_id)
-    try:
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        with open(path, "w", encoding="utf-8") as fh:
-            json.dump(data, fh)
-    except OSError:
-        pass
+    tezgah_store.put_doc("skill_pick", slug(session_id), data)
 
 
 def suggest(prompt, session_id=""):
@@ -617,31 +603,16 @@ def top_section(prompt, roots=None):
     return tuple(hit), covered, len(known)
 
 
-def _hint_path(session_id):
-    return os.path.join(tp.cache_dir(), "skill-section-hint",
-                        slug(session_id) + ".json")
-
-
 def _shown(session_id):
     """The sections this session was already pointed at, ["path:start"]."""
-    try:
-        with open(_hint_path(session_id), encoding="utf-8") as fh:
-            data = json.load(fh)
-    except (OSError, ValueError):
-        return []
+    data = tezgah_store.doc("section_hint", slug(session_id))
     return data if isinstance(data, list) else []
 
 
 def _remember_shown(session_id, shown):
-    """Bounded and best effort, like the judgement's cache: a dir that cannot
+    """Bounded and best effort, like the judgement's cache: a store that cannot
     be written costs a repeated line, never the turn."""
-    path = _hint_path(session_id)
-    try:
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        with open(path, "w", encoding="utf-8") as fh:
-            json.dump(shown[-KEPT:], fh)
-    except OSError:
-        pass
+    tezgah_store.put_doc("section_hint", slug(session_id), shown[-KEPT:])
 
 
 def section_hint(prompt, session_id="", host=None, roots=None):

package/hooks/tezgah_snapshot.py

@@ -24,11 +24,12 @@ import time
 import uuid
 
 import tezgah_integrity as ti
+import tezgah_store
 from tezgah_paths import cache_dir, root_for
 
 # A snapshot id is exactly what `capture` generates: 12 hex chars. Validated
 # before it is used as a path segment, so an id typed at a shell cannot walk out
-# of the store with `..` or an absolute path and read some other file's manifest.
+# of the store with `..` or an absolute path and read some other file as a blob.
 SNAPSHOT_ID = re.compile(r"\A[0-9a-f]{12}\Z")
 
 # The store is capped, because copying every file the agent touches with no
@@ -41,7 +42,9 @@ SNAPSHOT_ID = re.compile(r"\A[0-9a-f]{12}\Z")
 # mistake is most likely and most expensive, while the snapshot anyone actually
 # asks for is about a recent mistake - the oldest one is what nobody is coming
 # back for. Worst case on disk is CAP * MAX_BYTES = 400 MB; the realistic case
-# is a few MB, because source files are kilobytes.
+# is a few MB, because source files are kilobytes. Each blob is a file in its
+# own directory; the manifests are the `snapshots` rows of the cache database,
+# in capture order, and that order is what eviction reads.
 #
 # MAX_BYTES per file, above which nothing is captured at all. `capture` runs
 # synchronously in the gate a host calls before the write, so copying a
@@ -61,10 +64,10 @@ HASH_MAX_BYTES = 8 * MAX_BYTES
 
 
 def _store():
-    """Where the pre-write bytes go: beside the evidence ledger, in whichever
-    cache dir tezgah settled on for this environment (a sandboxed host has no
-    writable ~/.cache and falls back to temp), so all tezgah state sits under
-    one root."""
+    """Where the pre-write bytes go: beside the cache database that indexes
+    them, in whichever cache dir tezgah settled on for this environment (a
+    sandboxed host has no writable ~/.cache and falls back to temp), so all
+    tezgah state sits under one root."""
     return os.path.join(cache_dir(), "snapshots")
 
 
@@ -76,34 +79,13 @@ def _blob(snapshot_id):
     return os.path.join(_dir(snapshot_id), "file")
 
 
-def _manifest(snapshot_id):
-    return os.path.join(_dir(snapshot_id), "meta.json")
-
-
-def _dirs():
-    try:
-        return [os.path.join(_store(), n) for n in os.listdir(_store())]
-    except OSError:
-        return []
-
-
-def _evict(keep=None):
-    """Drop snapshot directories, oldest first, until `keep` (CAP) remain.
-
-    The order is each directory's own mtime, which is when its capture created
-    it - nothing writes inside a snapshot directory afterwards. Best effort: an
-    entry that cannot be read or removed costs the cap, never the capture the
-    gate is waiting on."""
-    keep = CAP if keep is None else keep
-    try:
-        # ns rather than the float mtime: two captures inside one tick must not
-        # evict an arbitrary one of the pair
-        entries = sorted((os.stat(p).st_mtime_ns, p) for p in _dirs())
-    except OSError:
-        return
-    for _, path in entries[:-keep]:
-        import shutil  # deferred: ~3.7 ms of import for a path only a write takes
-        shutil.rmtree(path, ignore_errors=True)
+def _drop(snapshot_ids):
+    """Remove the blob directories of `snapshot_ids`. Best effort: an entry
+    that cannot be removed costs disk, never the capture the gate is waiting
+    on."""
+    import shutil  # deferred: ~3.7 ms of import for a path only a write takes
+    for sid in snapshot_ids:
+        shutil.rmtree(_dir(sid), ignore_errors=True)
 
 
 def _hash_file(path):
@@ -222,21 +204,19 @@ def _capture_one(path, cwd, session_id):
 
 
 def _save(data, meta):
-    """Write one blob and its manifest under a fresh id, evict past CAP, and
-    return the id - or None, with nothing left behind, when the write failed."""
+    """Write one blob, index it with its manifest, evict past CAP, and return
+    the id - or None, with nothing left behind, when the write failed."""
     sid = uuid.uuid4().hex[:12]
     try:
         os.makedirs(_store(), mode=0o700, exist_ok=True)
         os.makedirs(_dir(sid), mode=0o700, exist_ok=True)
         with _private_open(_blob(sid)) as fh:
             fh.write(data)
-        with _private_open(_manifest(sid)) as fh:
-            fh.write(json.dumps(dict(meta, id=sid)).encode("utf-8"))
-    except OSError:
-        import shutil  # deferred: see _evict
-        shutil.rmtree(_dir(sid), ignore_errors=True)  # no half a snapshot
+        evicted = tezgah_store.add_snapshot(sid, dict(meta, id=sid), CAP)
+    except tezgah_store.ERRORS:
+        _drop([sid])  # no half a snapshot
         return None
-    _evict()
+    _drop(evicted)
     return sid
 
 
@@ -321,14 +301,13 @@ def restore(snapshot_id, force=False, expect=None):
     sid = str(snapshot_id or "")
     meta, data = None, b""
     if SNAPSHOT_ID.match(sid):
+        meta = next(iter(tezgah_store.snapshots(sid)), None)
         try:
-            with open(_manifest(sid), encoding="utf-8") as fh:
-                meta = json.load(fh)
             with open(_blob(sid), "rb") as fh:
                 data = fh.read()
-        except (OSError, ValueError):
+        except OSError:
             meta = None
-    if not meta:
+    if not isinstance(meta, dict):
         raise SnapshotError(
             "unknown snapshot id %r: the store holds no snapshot for it (CAP "
             "keeps the newest %d, and an evicted one is gone for good)"