@r1z4x/tezgah 1.3.0 → 1.5.0

package/hooks/tezgah_integrity.py

@@ -34,11 +34,12 @@ import time
 
 try:
     import fcntl
-except ImportError:  # not POSIX: the append stays unlocked, as it was before
+except ImportError:  # not POSIX: a legacy JSONL ledger was appended unlocked
     fcntl = None
 
 from tezgah_paths import (SWITCHES, _toplevel, cache_dir, latch, off, reply_lang,
                           root_for)
+import tezgah_store as ts
 
 # A command that actually checks the change, as opposed to one that merely runs.
 # Command position, like UI_CHECK: `echo pytest` and `cat pytest.ini` were
@@ -748,15 +749,6 @@ def _stored_text(value):
     return redact(str(value or ""))[:DETAIL_MAX]
 
 
-# How long an append waits for the lock before falling back to the unlocked
-# write it replaces. The holders are other hook processes appending one line, so
-# the wait is normally microseconds; the bound is what keeps a stuck holder from
-# wedging a hook, and the fallback is what keeps the row from being lost to the
-# lock that was meant to protect it.
-LOCK_WAIT = 1.0
-LOCK_POLL = 0.01
-
-
 def committed_size(handle):
     """The byte offset just past the last `\\n` in `handle`'s file, or 0 when the
     file holds no newline at all.
@@ -765,8 +757,8 @@ def committed_size(handle):
     record. A process killed inside a write leaves a trailing fragment that no
     newline ever terminated, and that fragment is not a record - it is the gap
     between what the writer meant to say and what reached the disk. The read
-    walks backwards in TAIL_CHUNK steps (the tail reader's own chunk below), so
-    its cost does not grow with a ledger that has run for a long session.
+    walks backwards in TAIL_CHUNK steps, so its cost does not grow with a file
+    that has run for a long session.
 
     `handle` is an open binary file object and is left positioned wherever the
     walk stopped: the callers truncate or append next, and neither reads the
@@ -795,54 +787,35 @@ def truncate_to_committed(handle):
 
 
 def _append(path, line):
-    """Append one line to `path` under an exclusive flock on the file itself.
-
-    Two writers reach one ledger file for real: a host fires PostToolUse once per
-    call of a parallel batch, each in its own process. Serialized, a row is
-    written by one writer at a time - the construction guarantee, not the
-    kernel's per-write atomicity on whichever filesystem the cache sits on. The
-    lock is the ledger's own descriptor, so no sidecar file appears beside it: a
-    reader lists that directory to find a session's ledger, and one extra name
-    per session would be a false record there. It dies with the process, so a
-    crash leaves nothing held.
-
-    A lock that cannot be taken within LOCK_WAIT is not taken, and the append
-    falls back to the write that was there before: no worse than the unlocked
-    path this replaced, and a busy lock never costs a row.
-
-    The torn tail is repaired before the write - under the lock when it is held,
-    and on the fallback too, because a fragment left in place would be terminated
-    by this very write and become the unparseable committed line the reader has
-    to refuse. The boundary is the last newline in the file, so the repair
-    removes a fragment and can never cut a record (the fallback's row says so:
-    `_unlocked`)."""
-    handle = None
+    """Append one row (`line`: its JSON and a newline) to the ledger at `path`.
+
+    The row goes into the cache's evidence database (`tezgah_store`) as one
+    INSERT in autocommit, so it is whole or absent: no torn tail, and two
+    writers queue on the database's lock instead of a file's. A row the
+    database cannot take (busy past its timeout during a sweep or a large
+    import, a damaged file) goes to the session's legacy JSONL file instead,
+    marked `unlocked` (`_jsonl_append`): the next read or write imports it, and
+    a lost `began` row would otherwise turn an honest pass into an orphan. Best
+    effort: a write that fails both ways is not fatal, as `note` promises."""
+    try:
+        ts.append_evidence(path, line.rstrip("\n"))
+    except ts.ERRORS:
+        _jsonl_append(path, _unlocked(line))
+
+
+def _jsonl_append(path, line):
+    """Append one line to the legacy JSONL file at `path`, without a lock (the
+    row says so: `_unlocked`), after cutting a torn tail back to the committed
+    boundary so the line is not merged into a fragment."""
     try:
-        # owner-only: a row carries commands and paths, and the default umask
-        # left the dir 0755 and the file 0644 (audit SEC-05 / L-6). The modes
-        # apply when the dir and the file are created.
+        # owner-only: a row carries commands and paths (audit SEC-05 / L-6)
         os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-        # binary: the boundary below is counted in bytes, not in characters
-        handle = os.fdopen(os.open(path, os.O_RDWR | os.O_APPEND | os.O_CREAT,
-                                   0o600), "a+b")
-        if fcntl is not None:
-            deadline = time.time() + LOCK_WAIT
-            while True:
-                try:
-                    fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                    break
-                except OSError:
-                    if time.time() >= deadline:
-                        line = _unlocked(line)
-                        break
-                    time.sleep(LOCK_POLL)
-        truncate_to_committed(handle)
-        handle.write(line.encode("utf-8"))
+        with os.fdopen(os.open(path, os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o600),
+                       "a+b") as handle:
+            truncate_to_committed(handle)
+            handle.write(line.encode("utf-8"))
     except OSError:
         pass
-    finally:
-        if handle is not None:
-            handle.close()  # flushes the line and releases the flock
 
 
 def note_path(path, kind, detail="", **fields):
@@ -884,22 +857,17 @@ def note(session_id, kind, detail="", **fields):
 
 
 def ledgers():
-    """Every evidence ledger, newest activity first.
+    """Every evidence ledger, newest activity first, as the paths
+    `events_path` reads: the sessions of this cache's evidence database, after
+    the bulk import of the legacy files (`tezgah_store.sessions`).
 
     The newest first is what a reader without a session id needs: the ledger a
     refusal was just written to is the one whose activity is newest."""
-
-    def mtime(path):
-        try:
-            return os.path.getmtime(path)
-        except OSError:
-            return 0.0
-
     d = os.path.join(cache_dir(), "evidence")
     try:
-        return sorted((os.path.join(d, n) for n in os.listdir(d)
-                       if n.endswith(".jsonl")), key=mtime, reverse=True)
-    except OSError:
+        return [os.path.join(d, session + ".jsonl")
+                for session, _ts in ts.sessions(cache_dir())]
+    except ts.ERRORS:
         return []
 
 
@@ -908,56 +876,21 @@ def kinds(session_id):
     return {str(e.get("kind")) for e in events(session_id) if e.get("kind")}
 
 
-# How much of the file one backwards read pulls in. The tail is a bounded number
-# of lines, so a chunk size only decides how many read() calls it takes.
+# How much of a file one backwards read pulls in (`committed_size`).
 TAIL_CHUNK = 8192
 
 
-def _tail_lines(path, n):
-    """The last `n` lines of a file, read by seeking from the end.
+def _ledger_lines(path, tail=None):
+    """The ledger's rows at `path` as text lines, each ending in a newline (the
+    shape `_parse` reads), oldest first; the last `tail` with one.
 
-    An unreadable file yields nothing: this is the reader the PreToolUse path
-    uses, and a gate that cannot see the ledger must let the call through."""
+    A ledger that cannot be read yields nothing: this is the reader the
+    PreToolUse path uses, and a gate that cannot see the ledger must let the
+    call through."""
     try:
-        with open(path, "rb") as fh:
-            fh.seek(0, os.SEEK_END)
-            pos = fh.tell()
-            data = b""
-            while pos > 0 and data.count(b"\n") <= n:
-                step = min(TAIL_CHUNK, pos)
-                pos -= step
-                fh.seek(pos)
-                data = fh.read(step) + data
-    except OSError:
+        return [text + "\n" for text in ts.evidence_rows(path, tail)]
+    except ts.ERRORS:
         return []
-    # `keepends`: whether the last line was terminated is what `_parse` reads to
-    # tell a torn tail from a committed record, and splitting it away here would
-    # leave every line looking unterminated.
-    return _lines(data)[-n:]
-
-
-def _lines(data):
-    """A ledger's bytes as text lines, ends kept, split on b"\\n" alone.
-
-    `str.splitlines` also splits on U+2028, U+2029 and U+0085, which a JSON
-    writer may leave raw inside a string (opencode's JSON.stringify does), so an
-    honest row read as two damaged ones. A line that is not UTF-8 becomes a
-    marker no JSON reader parses, so `_parse` names it as damage; decoding the
-    whole file strictly raised instead, and the Stop rule failed open."""
-    parts = data.split(b"\n")
-    out = []
-    for i, part in enumerate(parts):
-        if i < len(parts) - 1:
-            part += b"\n"
-        elif not part:
-            break
-        try:
-            out.append(part.decode("utf-8"))
-        except UnicodeDecodeError:
-            out.append("\x00not utf-8 %s%s" % (
-                hashlib.sha1(part).hexdigest()[:12],
-                "\n" if part.endswith(b"\n") else ""))
-    return out
 
 
 # The kind a damaged ledger line is recorded under: one row per damaged line,
@@ -976,15 +909,11 @@ _DAMAGE_SEEN = set()
 
 def _note_damage(path, key, line):
     """Write the one `ledger_damage` row for a damaged line, unless the ledger
-    already holds it. Damage is rare, so the dedup reads the whole file."""
+    already holds it. Damage is rare, so the dedup reads the whole ledger."""
     if (path, key) in _DAMAGE_SEEN:
         return
     _DAMAGE_SEEN.add((path, key))
-    try:
-        with open(path, encoding="utf-8", errors="replace") as fh:
-            if any(DAMAGE_KIND in held and key in held for held in fh):
-                return
-    except OSError:
+    if any(DAMAGE_KIND in held and key in held for held in _ledger_lines(path)):
         return
     note_path(path, DAMAGE_KIND, cut(line.strip(), 80), key=key)
 
@@ -1045,21 +974,14 @@ def events_path(path, tail=None):
     """Every parseable ledger entry at an explicit ledger path, oldest first.
     `events` is this function with the path derived from a session id. Every
     `verify_ok` is paired with the gate's `began` row on the way out (`_pair`)."""
-    if tail:
-        lines = _tail_lines(path, tail)
-    else:
-        try:
-            with open(path, "rb") as fh:
-                lines = _lines(fh.read())
-        except OSError:
-            return []
+    lines = _ledger_lines(path, tail)
     return _pair(_parse(lines, path), path, lines)
 
 
 def events(session_id, tail=None):
     """Every parseable ledger entry for this session, oldest first.
 
-    With `tail`, only the last `tail` lines are read. The file grows with the
+    With `tail`, only the last `tail` rows are read. The ledger grows with the
     session and the gate reads it on every gated call, so the tail path must
     never parse the whole of it."""
     return events_path(_path(session_id), tail)
@@ -1100,11 +1022,7 @@ def turn_rows(session_id, turns=False, agent=None):
     taint another's effects (security-09). The parent (`agent` None) keeps the
     whole turn, its subagents' work included - that work is the parent's turn."""
     path = _path(session_id)
-    try:
-        with open(path, "rb") as fh:
-            lines = _lines(fh.read())
-    except OSError:
-        return ([], 0) if turns else []
+    lines = _ledger_lines(path)
     start = _turn_line(lines)
     rows = _pair(_parse(lines[start:], path), path, lines, lines[:start])
     if agent:
@@ -1194,13 +1112,6 @@ def prior_calls(session_id, digest, tail=200, agent=None):
             turn[-1].get("fail_class"))
 
 
-# How much of a sibling session's ledger a cross-session read parses. A write
-# inside the window is that ledger's newest activity by definition of "inside",
-# so the tail is where it is; a session whose window-write sits further back
-# than this many rows is missed.
-WRITE_TAIL = 200
-
-
 def _abs_target(path, cwd):
     """`path` as an absolute real path, resolved against `cwd` when relative.
 
@@ -1276,11 +1187,9 @@ def writers_elsewhere(path, session_id, minutes=10, cwd=None):
     back to a reader that takes a session id (`_path` would slug it a second
     time and open another file).
 
-    Neutral value: [] when the cache cannot be listed; a ledger that cannot be
-    read or holds a damaged committed line costs only its own rows
-    (`_foreign_rows`, audit GAP-02 / M-7). A cross-session rule that cannot see
-    the other ledgers has learned nothing, and must stay silent rather than act
-    on a guess.
+    Neutral value: [] when the ledger cannot be read; a row that does not
+    parse is skipped. A cross-session rule that cannot see the other ledgers has
+    learned nothing, and must stay silent rather than act on a guess.
 
     ponytail: a shell write counts only through the target its `run` row
     carries (a redirect or `tee`, `tezgah_gate.write_paths`): a sibling that
@@ -1289,38 +1198,23 @@ def writers_elsewhere(path, session_id, minutes=10, cwd=None):
     want = _abs_target(path, cwd)
     if not want:
         return []
-    d = os.path.join(cache_dir(), "evidence")
-    mine = _slug(session_id) + ".jsonl"
     now = time.time()
-    window = minutes * 60
-    found = []
+    found = {}
     try:
-        entries = list(os.scandir(d))
-    except OSError:
+        recent = ts.recent_rows(cache_dir(), now - minutes * 60, ("edit", "run"),
+                                _slug(session_id))
+    except ts.ERRORS:
         return []
-    for entry in entries:
-        if not entry.name.endswith(".jsonl") or entry.name == mine:
-            continue
+    # the `ts` column, not the row's own field: a row inserted outside the
+    # hooks need not carry one, and nothing here may raise out of the gate
+    for stem, stamp, text in recent:
         try:
-            # nothing in a file whose last write is older than the window can
-            # be inside it, so most of a long-lived cache is skipped unread
-            if now - entry.stat().st_mtime > window:
-                continue
-        except OSError:
+            row = json.loads(text)
+        except ValueError:
             continue
-        newest = None
-        for row in _foreign_rows(entry.path, WRITE_TAIL):
-            if row.get("kind") not in ("edit", "run"):
-                continue
-            ts = row.get("ts")
-            if not isinstance(ts, (int, float)) or now - ts > window:
-                continue
-            if row.get("target") != want:
-                continue
-            newest = ts if newest is None else max(newest, ts)
-        if newest is not None:
-            found.append((newest, entry.name[:-len(".jsonl")]))
-    return [stem for _, stem in sorted(found, key=lambda pair: -pair[0])]
+        if isinstance(row, dict) and row.get("target") == want:
+            found[stem] = max(found.get(stem, stamp), stamp)
+    return sorted(found, key=lambda stem: -found[stem])
 
 
 def note_turn(session_id, prompt, workspace=None):
@@ -5055,10 +4949,12 @@ def changed_files_notice(session_id, base=None):
 
 
 # --- pairing: a pass the gate never saw begin (plan 051 part 8) --------------
-# The field `_append` adds to a row it wrote without the lock, and the in-memory
-# mark `_pair` puts on a `verify_ok` no gate-written `began` row answers. The
-# mark is never written: `passing_check` refuses a marked row, and the Stop rule
-# reads one in the turn as "evidence tampered".
+# The field a JSONL append adds to a row it wrote without the lock (`_append`'s
+# fallback when the database refuses a row, or the old file writer's LOCK_WAIT
+# fallback, whose imported rows may still carry it), and the in-memory mark
+# `_pair` puts on a `verify_ok` no gate-written `began` row answers. The mark is
+# never written: `passing_check` refuses a marked row, and the Stop rule reads
+# one in the turn as "evidence tampered".
 UNLOCKED = "unlocked"
 ORPHAN = "orphan"
 ORPHANED = (
@@ -5075,7 +4971,7 @@ PAIR_SEED = 200
 
 
 def _unlocked(line):
-    """`line` with `unlocked: 1` added to its row (`_append`'s LOCK_WAIT
+    """`line` with `unlocked: 1` added to its row (`_append`'s JSONL
     fallback), or as it was when it is not one row."""
     try:
         row = json.loads(line)
@@ -5106,8 +5002,11 @@ def _pair(rows, path, lines, before=()):
     - in a turn holding a `crash` row: the gate that writes `began` may be the
       code that crashed;
     - with `pretooluse-off` armed: no gate ran;
-    - when an append took `_append`'s unlocked fallback, or this platform has
-      no lock at all: that truncate can cut a concurrent writer's `began` row;
+    - when a row says a JSONL append wrote it without the lock (UNLOCKED:
+      `_append`'s fallback when the database refused a row, or the old file
+      writer's), or this platform has no flock, so a legacy file's appends
+      were all unlocked: that append's truncate can have cut a concurrent
+      writer's `began` row, and a row the database refused may be one;
     - for a ledger in the sandbox fallback cache, or one whose session also has
       a ledger in the other cache dir: a sandboxed host can split one call's two
       rows over two files (`tezgah_context`'s fallback-cache note).
@@ -5185,4 +5084,4 @@ def _unpaired_exempt(path, lines):
     fallback = os.path.realpath(os.path.join(fallback_cache(), "evidence"))
     dirs = {os.path.realpath(os.path.join(CACHE, "evidence")), fallback} - {here}
     return here == fallback or any(
-        os.path.exists(os.path.join(d, os.path.basename(path))) for d in dirs)
+        ts.has_evidence(os.path.join(d, os.path.basename(path))) for d in dirs)