@r1z4x/tezgah 1.2.0 → 1.4.0

package/hooks/tezgah_taste.py

@@ -6,8 +6,8 @@ Opt-in and off by default. A row is written only when the `taste-on` marker is
 armed (`tezgah_paths.armed`), the repository carries no `.no-taste` mark, its
 `.tezgah/` already exists and is the user's own (`workspace_from_repo`: a
 workspace that came with the clone is data, and nothing of the user's is
-written into it). The store is `<git root>/.tezgah/taste/signals.jsonl`, one
-JSON object per line, owner-only.
+written into it). The store is the `signals` table of
+`<git root>/.tezgah/taste/taste.db` (`tezgah_store.taste`), owner-only.
 
 Every public function is total: it never raises and makes no network call.
 The capture functions append at most one row each. They run on the prompt and
@@ -15,15 +15,24 @@ PostToolUse hot paths, so the switch is read before anything else is.
 
 Learning lives in `bin/tezgah-taste learn` and `tezgah_taste_ledger`; the one
 thing fed back from here is `write_note`, the learnings in scope for a write.
+`learn_later` starts that `learn` in the background at session start.
 """
 import datetime
 import json
 import os
+import re
+import subprocess
+import sys
+import time
 
 import tezgah_integrity as ti
+import tezgah_paths
 from tezgah_paths import _toplevel, armed, root_for, workspace_from_repo
 
 ARM = "taste-on"
+LEARN_EVERY = 3600  # seconds between two background `learn` starts per repo
+BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
+                   "bin", "tezgah-taste")
 MARK = ".no-taste"
 VERSION = 1
 PROMPT_MAX = 2000
@@ -77,9 +86,10 @@ def _write(root, kind, session_id, host, **fields):
            .strftime("%Y-%m-%dT%H:%M:%SZ"),
            "kind": kind, "session": str(session_id or ""), "host": host or _host()}
     row.update(fields)
-    # the ledger's own writer: its flock, torn-tail repair and 0700/0600 modes
-    ti._append(os.path.join(root, ".tezgah", "taste", "signals.jsonl"),
-               json.dumps(row, ensure_ascii=False) + "\n")
+    # imported here, not at the top: the off path never loads sqlite3
+    import tezgah_store as store
+    with store.taste(os.path.join(root, ".tezgah", "taste")) as db:
+        store.append(db, "signals", row)
 
 
 def _host():
@@ -203,6 +213,50 @@ def write_note(session_id, inp, cwd, tool=None):
         return ""
 
 
+def learn_later(root):
+    """Start `tezgah-taste learn --repo <root> --no-fallback` detached when the
+    signals of `root` (the git root `enabled` returned) grew since the last
+    start and an hour has passed; the stamp (a `taste_learn` document of the
+    cache database) keeps the newest signal's append number. Only when
+    TypeSafe resolves: the automatic run never pays a
+    generative provider for a decision it may not apply. Not on a
+    machine without flock (Windows), where two background runs in two
+    repositories would race on the user ledger. ponytail: no Windows lock; add
+    one (msvcrt.locking) before enabling it there. Total; the stamp is written
+    before the spawn, so a failed start waits an hour."""
+    try:
+        if not root:
+            return
+        import tezgah_store as store
+        with store.taste(os.path.join(root, ".tezgah", "taste"), create=False) as db:
+            if db is None:
+                return
+            count = store.last(db, "signals")
+        base = os.path.join(tezgah_paths.cache_dir(), "taste-learn")
+        name = re.sub(r"[^A-Za-z0-9]+", "-", root).strip("-")
+        last = store.doc("taste_learn", name)
+        last = last if isinstance(last, dict) else {}
+        now = time.time()
+        if count <= last.get("rows", 0) or now - last.get("at", 0) < LEARN_EVERY:
+            return
+        # the costlier checks run only once a start is due: at most hourly
+        import tezgah_judge
+        import tezgah_taste_ledger
+        if tezgah_taste_ledger.fcntl is None or tezgah_paths.off("judge-off") \
+                or not tezgah_judge.named(("typesafe",)):
+            return
+        if not store.put_doc("taste_learn", name, {"rows": count, "at": now}):
+            return
+        os.makedirs(base, exist_ok=True)
+        with open(os.path.join(base, name + ".log"), "ab") as log:
+            subprocess.Popen([sys.executable, BIN, "learn", "--repo", root,
+                              "--no-fallback"],
+                             cwd=root, stdin=subprocess.DEVNULL, stdout=log,
+                             stderr=log, start_new_session=True)
+    except Exception:
+        pass
+
+
 def main(argv):
     """`tezgah_taste.py '<json>'`: `note_write`, then `write_note`, for a host
     that cannot call them in process (opencode's plugin is JavaScript). The one
@@ -223,5 +277,4 @@ def main(argv):
 
 
 if __name__ == "__main__":
-    import sys
     sys.exit(main(sys.argv[1:]))

package/hooks/tezgah_taste_ledger.py

@@ -21,20 +21,20 @@ answers and keeps the ledger.
 - Apply: a rule needs confidence >= `RULE_AT` and a calibration bound >=
   `BOUND_AT` (Clopper-Pearson on an independently labelled sample); a hint needs
   `HINT_AT`; a learning whose evidence paths are all gone from the tree is
-  suspended. The benefit gate's stop file turns injection off.
+  suspended. The benefit gate's stopped state turns injection off.
 
-Store: `<repo>/.tezgah/taste/ledger.json` for path, language and repository
-learnings, `~/.config/tezgah/taste/ledger.json` for user learnings.
+Store: `tezgah_store.taste` - `<repo>/.tezgah/taste/taste.db` for path,
+language and repository learnings and every taste row, and
+`~/.config/tezgah/taste/taste.db` for user learnings.
 """
 import datetime
 import errno
 import fnmatch
-import json
 import math
 import os
 
-import tezgah_integrity as ti
-from tezgah_paths import CONFIG_DIR, cache_dir
+import tezgah_store as store
+from tezgah_paths import CONFIG_DIR
 
 try:
     import fcntl
@@ -66,35 +66,21 @@ def user_dir():
     return os.path.join(CONFIG_DIR, "taste")
 
 
-def _load(path):
-    try:
-        with open(path, encoding="utf-8") as fh:
-            data = json.load(fh)
-        if isinstance(data, dict) and isinstance(data.get("learnings"), dict):
-            data.setdefault("meta", {})
-            return data
-    except (OSError, ValueError):
-        pass
-    return {"v": 1, "learnings": {}, "meta": {}}
-
-
-def _save(path, data):
-    os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-    tmp = "%s.%d.tmp" % (path, os.getpid())
-    with open(tmp, "w", encoding="utf-8") as fh:
-        json.dump(data, fh, ensure_ascii=False, indent=1, sort_keys=True)
-    os.replace(tmp, path)
+def _load(dirpath, user=False):
+    with store.taste(dirpath, user, create=False) as db:
+        return store.load_ledger(db) if db else {"v": 1, "learnings": {}, "meta": {}}
 
 
 def load(root):
     """{"repo": ledger, "user": ledger}."""
-    return {"repo": _load(os.path.join(store_dir(root), "ledger.json")),
-            "user": _load(os.path.join(user_dir(), "ledger.json"))}
+    return {"repo": _load(store_dir(root)), "user": _load(user_dir(), user=True)}
 
 
 def save(root, ledgers):
-    _save(os.path.join(store_dir(root), "ledger.json"), ledgers["repo"])
-    _save(os.path.join(user_dir(), "ledger.json"), ledgers["user"])
+    with store.taste(store_dir(root)) as db:
+        store.save_ledger(db, ledgers["repo"])
+    with store.taste(user_dir(), user=True) as db:
+        store.save_ledger(db, ledgers["user"])
 
 
 def every(ledgers):
@@ -142,34 +128,24 @@ def cp_lower(x, n, alpha=0.05):
     return lo
 
 
-def rows(path):
-    try:
-        with open(path, encoding="utf-8") as fh:
-            for line in fh:
-                try:
-                    row = json.loads(line)
-                except ValueError:
-                    continue
-                if isinstance(row, dict):
-                    yield row
-    except OSError:
-        return
+def rows(root, table, where="", args=()):
+    """One taste table's rows in append order (`tezgah_store.TASTE_ROWS`); []
+    when the repository has no store yet."""
+    with store.taste(store_dir(root), create=False) as db:
+        return store.rows(db, table, where, args) if db else []
 
 
-def append(path, row, strict=False):
-    """One row through the ledger's own writer: flock, torn-tail repair, 0600.
-    That writer drops a row it cannot write, which a hook wants; `strict` (the
-    CLI's decision, defect and label rows) reads the tail back and raises
-    OSError when the row is not there, so `learn` stops before it saves a
-    ledger whose decision was never recorded and would be re-applied."""
-    line = json.dumps(row, ensure_ascii=False) + "\n"
-    ti._append(path, line)
-    if strict:
-        data = line.encode("utf-8")
-        with open(path, "rb") as fh:
-            fh.seek(max(0, os.path.getsize(path) - len(data)))
-            if fh.read() != data:
-                raise OSError("taste row not written to %s" % path)
+def append(root, table, row, strict=False):
+    """One row into a taste table. A hook drops a row it cannot write; `strict`
+    (the CLI's decision, defect and label rows) raises instead, so `learn` stops
+    before it saves a ledger whose decision was never recorded and would be
+    re-applied."""
+    try:
+        with store.taste(store_dir(root)) as db:
+            store.append(db, table, row)
+    except store.ERRORS:
+        if strict:
+            raise
 
 
 class Busy(Exception):
@@ -220,8 +196,8 @@ def calibration(root):
     bound on that precision. Only typed-provider decisions count; the labels are
     a sample drawn independently of what was injected (`tezgah-taste label`)."""
     x = n = 0
-    for row in rows(os.path.join(store_dir(root), "labels.jsonl")):
-        if row.get("provider") != "typesafe" or row.get("decided") != "preference":
+    for row in rows(root, "labels", "WHERE provider = ?", ("typesafe",)):
+        if row.get("decided") != "preference":
             continue
         n += 1
         x += row.get("label") == "preference"
@@ -428,11 +404,8 @@ def suspended(learning, root):
 
 def stopped(root):
     """True when the benefit gate stopped injection for this repository."""
-    try:
-        with open(os.path.join(store_dir(root), "gate.json"), encoding="utf-8") as fh:
-            return bool(json.load(fh).get("stopped"))
-    except (OSError, ValueError, AttributeError):
-        return False
+    with store.taste(store_dir(root), create=False) as db:
+        return bool(db) and store.gate_stopped(db)
 
 
 def usable(root, day=None):
@@ -478,10 +451,10 @@ def block(root, day=None):
 
 
 def record_injection(root, session_id, ids, day=None):
-    """One `injected.jsonl` row: the meta loop counts a later correction in this
+    """One `injected` row: the meta loop counts a later correction in this
     session against these ids, and the benefit gate splits its arms at the first."""
     if ids:
-        append(os.path.join(store_dir(root), "injected.jsonl"),
+        append(root, "injected",
                {"session": str(session_id or ""), "day": now_day() if day is None else day,
                 "ids": list(ids)})
 
@@ -489,9 +462,8 @@ def record_injection(root, session_id, ids, day=None):
 def injected(root, session_id):
     """The learning ids injected in one session."""
     ids = set()
-    for row in rows(os.path.join(store_dir(root), "injected.jsonl")):
-        if row.get("session") == session_id:
-            ids.update(row.get("ids") or ())
+    for row in rows(root, "injected", "WHERE session = ?", (session_id,)):
+        ids.update(row.get("ids") or ())
     return ids
 
 
@@ -502,24 +474,18 @@ def write_note(root, session_id, rel_path):
     if not session_id or not rel_path or stopped(root):
         return ""
     kind = os.path.splitext(rel_path)[1] or os.path.basename(rel_path)
-    seen_path = os.path.join(cache_dir(), "taste-notes",
-                             "%s.json" % str(session_id).replace(os.sep, "_"))
-    try:
-        with open(seen_path, encoding="utf-8") as fh:
-            seen = set(json.load(fh))
-    except (OSError, ValueError, TypeError):
-        seen = set()
+    session_id = str(session_id)
+    with store.taste(store_dir(root), create=False) as db:
+        seen = store.seen(db, session_id) if db else set()
     hits = [(mode, learning) for mode, learning in usable(root)
             if learning["scope"] in ("path", "language") and learning["id"] not in seen
             and fnmatch.fnmatch(rel_path, learning["where"])]
     if not hits:
         return ""
-    seen.update(m[1]["id"] for m in hits)
     try:
-        os.makedirs(os.path.dirname(seen_path), mode=0o700, exist_ok=True)
-        with open(seen_path, "w", encoding="utf-8") as fh:
-            json.dump(sorted(seen), fh)
-    except OSError:
+        with store.taste(store_dir(root)) as db:
+            store.mark_seen(db, session_id, [m[1]["id"] for m in hits])
+    except store.ERRORS:
         return ""
     record_injection(root, session_id, [m[1]["id"] for m in hits])
     return ("Taste for %s files (narrower scope wins):\n" % kind
@@ -529,8 +495,8 @@ def write_note(root, session_id, rel_path):
 # --- markdown and export ----------------------------------------------------------------
 
 def markdown(root, day=None):
-    """Rewrite `<store>/<category>/taste.md` from the active learnings; the JSON
-    ledger stays the source of truth. Returns the files written."""
+    """Rewrite `<store>/<category>/taste.md` from the active learnings; the
+    database stays the source of truth. Returns the files written."""
     day = now_day() if day is None else day
     by_cat = {}
     for learning in every(load(root)).values():
@@ -590,7 +556,7 @@ def export_agents(root):
 
 def first_injection(root):
     """The day of the first injection, or None: where the gate's arms split."""
-    days = [r["day"] for r in rows(os.path.join(store_dir(root), "injected.jsonl"))
+    days = [r["day"] for r in rows(root, "injected")
             if isinstance(r.get("day"), (int, float))]
     return min(days) if days else None
 
@@ -601,13 +567,13 @@ def gate(root, need=980, alpha=0.1):
     Every decided signal is one prompt after a writing turn. The arms split at
     the first injection, by when the turn happened (`at`), not by when `learn`
     decided it, so a backfill of old sessions stays in the before arm. Once both
-    arms hold `need` turns, `gate.json` says stopped unless the after-rate is
+    arms hold `need` turns, the gate's state says stopped unless the after-rate is
     lower at one-sided p < `alpha`. Returns the report."""
     first = first_injection(root)
     arms = {"before": [0, 0], "after": [0, 0]}
-    for row in rows(os.path.join(store_dir(root), "decisions.jsonl")):
+    for row in rows(root, "decisions", "WHERE provider = ?", ("typesafe",)):
         at = row.get("at")
-        if row.get("provider") != "typesafe" or not isinstance(at, (int, float)):
+        if not isinstance(at, (int, float)):
             continue
         arm = arms["after" if first is not None and at >= first else "before"]
         arm[0] += 1
@@ -623,6 +589,6 @@ def gate(root, need=980, alpha=0.1):
         report["p"] = round(0.5 * math.erfc(z / math.sqrt(2)), 4)
     if n1 >= need and n2 >= need:
         report["stopped"] = not (report["p"] is not None and report["p"] < alpha)
-        _save(os.path.join(store_dir(root), "gate.json"),
-              {"stopped": report["stopped"], "report": report})
+        with store.taste(store_dir(root)) as db:
+            store.set_gate(db, report["stopped"], report)
     return report

package/hooks/tezgah_update.py

@@ -22,6 +22,7 @@ import sys
 import time
 
 import tezgah_paths as tp
+import tezgah_store
 
 REPO = os.environ.get("TEZGAH_REPO") or "r1z4x/tezgah"
 CHECK_EVERY = 24 * 3600
@@ -30,10 +31,6 @@ SEMVER = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
 ARROW = "\u2191"  # ↑: East Asian width Ambiguous, one cell, never an emoji
 
 
-def cache_path():
-    return os.path.join(tp.cache_dir(), "update.json")
-
-
 def endpoint():
     """Where the newest release is read, repointed by TEZGAH_UPDATE_URL (tests).
 
@@ -73,32 +70,15 @@ def disabled():
 
 
 def read_cache():
-    try:
-        with open(cache_path(), encoding="utf-8") as fh:
-            data = json.load(fh)
-        return data if isinstance(data, dict) else {}
-    except Exception:
-        return {}
+    """The `update_check` store's one document, `{checked, latest}`, or {}."""
+    data = tezgah_store.doc("update_check", "")
+    return data if isinstance(data, dict) else {}
 
 
 def write_cache(data):
-    """Write the cache whole, owner-only, through a rename, so a redraw reading it
+    """Write the cache whole, in one statement, so a redraw reading it
     mid-write sees the old answer or the new one and never half of either."""
-    path = cache_path()
-    tmp = "%s.%d.tmp" % (path, os.getpid())
-    try:
-        os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
-        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
-        with os.fdopen(fd, "w", encoding="utf-8") as fh:
-            json.dump(data, fh)
-        os.replace(tmp, path)
-        return True
-    except OSError:
-        try:
-            os.remove(tmp)
-        except OSError:
-            pass
-        return False
+    return tezgah_store.put_doc("update_check", "", data)
 
 
 # Every 0.x ever published, retired by 1.0.0: npm served 0.17.0 through 0.32.0

package/hosts/cursor/hook.py

@@ -52,7 +52,8 @@ try:
     from tezgah_integrity import (  # noqa: E402
         SUBAGENT_CHANNEL, bind_session, note, note_tool, ran_nothing, report_bytes,
         stop_reason)
-    from tezgah_paths import cache_dir, off  # noqa: E402
+    from tezgah_paths import off  # noqa: E402
+    import tezgah_store  # noqa: E402
     from tezgah_taste import write_note  # noqa: E402
     from tezgah_untrusted import marks  # noqa: E402
 except Exception as exc:
@@ -215,25 +216,21 @@ def first_time(session_id, tag):
     """True the first time a tag is seen for a session; fail-open when unwritable."""
     if not session_id:
         return True
-    mark = os.path.join(cache_dir(), "reinforced", slug(str(session_id)), tag)
-    if os.path.exists(mark):
-        return False
     try:
-        os.makedirs(os.path.dirname(mark), exist_ok=True)
-        open(mark, "w", encoding="utf-8").close()
-    except OSError:
-        pass
-    return True
+        return tezgah_store.mark("cursor_reinforced", "%s/%s" % (slug(str(session_id)), tag))
+    except tezgah_store.ERRORS:
+        return True
 
 
-def answer_path(session_id):
-    """Where the last assistant text of a conversation is kept.
+def answer_key(session_id):
+    """The key of the last assistant text of a conversation in the
+    `cursor_answer` store.
 
     Cursor's `stop` payload carries only `status`/`loop_count`, so the reply the
     Stop rule has to read arrives on the earlier `afterAgentResponse`, which
-    documents `{"text": "<assistant final text>"}`. One file per conversation,
-    overwritten."""
-    return os.path.join(cache_dir(), "answer", slug(str(session_id or "nosession")))
+    documents `{"text": "<assistant final text>"}`. One document per
+    conversation, overwritten."""
+    return slug(str(session_id or "nosession"))
 
 
 # A long reply is kept by its two ends rather than by one slice: the Stop rule's
@@ -255,29 +252,20 @@ def remember_answer(session_id, text):
     count of what was dropped, then `ANSWER_TAIL` characters of its tail - so
     both the opener the Stop rule reads and the closing marker it must not lose
     survive the round trip. A reply with no `conversation_id` is not stored; a
-    file that cannot be written is not fatal."""
+    store that cannot be written is not fatal."""
     if not session_id or not text:
         return
-    path = answer_path(session_id)
-    try:
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        text = str(text)
-        dropped = len(text) - ANSWER_HEAD - ANSWER_TAIL
-        if dropped > 0:
-            text = text[:ANSWER_HEAD] + ANSWER_CUT % dropped + text[-ANSWER_TAIL:]
-        with open(path, "w", encoding="utf-8") as fh:
-            fh.write(text)
-    except OSError:
-        pass
+    text = str(text)
+    dropped = len(text) - ANSWER_HEAD - ANSWER_TAIL
+    if dropped > 0:
+        text = text[:ANSWER_HEAD] + ANSWER_CUT % dropped + text[-ANSWER_TAIL:]
+    tezgah_store.put_doc("cursor_answer", answer_key(session_id), text)
 
 
 def last_answer(session_id):
     """The reply remembered for this conversation, or ""."""
-    try:
-        with open(answer_path(session_id), encoding="utf-8") as fh:
-            return fh.read()
-    except OSError:
-        return ""
+    text = tezgah_store.doc("cursor_answer", answer_key(session_id))
+    return text if isinstance(text, str) else ""
 
 
 def dispatch(payload):