@r1z4x/tezgah 1.6.0 → 1.7.0

package/hooks/tezgah_stopspec.py

@@ -102,6 +102,13 @@ NOT_COUNTED = (
                     "constants over one trace, read off the reply, not a row"),
     ("evidence tampered", "a trace-validity guard in the selector, before the fold"),
 )
+# Plan 080 (2026-10-11), after the pre-registration and not a change to it: the
+# admission ("doğrulanmadı") is applied by the selector to whichever fold it is
+# given (`tezgah_integrity._admitted`: a check-repair class cleared after a check
+# that failed since the newest change), and `residual work` is a selector verdict
+# asked at every exit that would end the turn (`_residual_block`). Both reach the
+# spec's verdict through the shared selector, so neither is a formula here.
+
 # GO 2's live floor: shadow rows needed before the 99% bar is read. At 400 a
 # 99% rate is 4 disagreements, so one more or less moves it by 0.25 pp.
 MIN_LIVE_N = 400

package/hooks/tezgah_store.py

@@ -269,17 +269,58 @@ def _taste_put(table):
     return put
 
 
+def _recorded(conn, name, st):
+    """(the bytes `imported` counts for the file `name` with stat `st`, the
+    device id of the row they were found under when it is not the current one)
+    or None. A volume remount can renumber the device of a file whose name and
+    inode are unchanged, and a store from before the fix imported such a file
+    again from offset 0 under the new id, so every row for (name, inode) is one
+    import stream. The longest is the file's offset under any id - this id's
+    own row included, when the id comes back to a shorter stream - so no byte
+    goes in twice."""
+    found = conn.execute("SELECT bytes, dev FROM imported WHERE name = ? AND ino = ? "
+                         "ORDER BY bytes DESC, dev = ? DESC LIMIT 1",
+                         (name, st.st_ino, st.st_dev)).fetchone()
+    if found is None:
+        return None
+    return found[0], None if found[1] == st.st_dev else found[1]
+
+
+def _record(conn, key, start, end, stale):
+    """The file `key` in up to byte `end`, its import having resumed at `start`.
+    An import resumed from a row under another device id (`stale`) moves that
+    row to the current key, and a shorter stream's row already under the
+    current key takes the stale id: the two swap, so one row per import stream
+    stays, which is what the doctor's repair counts a double import by
+    (`tezgah-doctor --repair-evidence`). A file cut below what went in (a row
+    past `start`) no longer holds what its imports read: it goes on as one
+    stream, which resumes at its end and which the repair leaves alone."""
+    name, dev, ino = key
+    if conn.execute("SELECT 1 FROM imported WHERE name = ? AND ino = ? AND bytes > ?",
+                    (name, ino, start)).fetchone():
+        conn.execute("DELETE FROM imported WHERE name = ? AND ino = ?", (name, ino))
+    elif stale is not None:
+        conn.execute("DELETE FROM imported WHERE name = ? AND dev = ? AND ino = ?",
+                     (name, stale, ino))
+        conn.execute("UPDATE imported SET dev = ? WHERE name = ? AND dev = ? AND ino = ?",
+                     (stale, name, dev, ino))
+    conn.execute("INSERT OR REPLACE INTO imported (name, dev, ino, bytes) "
+                 "VALUES (?, ?, ?, ?)", key + (end,))
+
+
 def _offset(conn, name, fh):
-    """(the file's key, the byte offset its import resumes at, whether that
-    differs from the recorded one). A file cut below what went in (rewritten in
-    place) resumes at its end, so the rows appended after the cut still
-    import."""
+    """(the file's key, the byte offset its import resumes at, whether the
+    recorded row must be rewritten, the device id of the row it resumes from
+    when that is not the current one (`_recorded`)). The row is rewritten when
+    its offset differs or it is under another id: a file cut below what went
+    in (rewritten in place) resumes at its end, so the rows appended after the
+    cut still import."""
     st = os.fstat(fh.fileno())
     key = (name, st.st_dev, st.st_ino)
-    found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? "
-                         "AND ino = ?", key).fetchone()
+    found = _recorded(conn, name, st)
     start = min(found[0], st.st_size) if found else 0
-    return key, start, bool(found and found[0] != start)
+    stale = found[1] if found else None
+    return key, start, bool(found and found[0] != start) or stale is not None, stale
 
 
 def _import_rows(conn, name, src, st, put=None, keep=False):
@@ -303,14 +344,14 @@ def _import_rows(conn, name, src, st, put=None, keep=False):
         _failed(conn, name, st)
         return
     with fh:  # closing it releases the flock
-        key, start, cut = _offset(conn, name, fh)
+        key, start, cut, _stale = _offset(conn, name, fh)
         fh.seek(start)
         raw = fh.read()
         if b"\n" in raw or cut:
             with transaction(conn):
                 if not _locked(fh):
                     return  # its writer holds it: the next open takes it
-                key, start, cut = _offset(conn, name, fh)
+                key, start, cut, stale = _offset(conn, name, fh)
                 fh.seek(start)
                 raw = fh.read()
                 whole = raw[:raw.rfind(b"\n") + 1]
@@ -318,8 +359,7 @@ def _import_rows(conn, name, src, st, put=None, keep=False):
                     put = put or _taste_put(name[:-len(".jsonl")])
                     for line in whole.split(b"\n"):
                         put(conn, line)
-                    conn.execute("INSERT OR REPLACE INTO imported (name, dev, ino, bytes) "
-                                 "VALUES (?, ?, ?, ?)", key + (start + len(whole),))
+                    _record(conn, key, start, start + len(whole), stale)
         # aside only when every byte went in (no fragment past the last newline)
         # and no writer appended past the read (the old writer appends unlocked
         # once its own wait runs out), under the flock so no writer is inside
@@ -523,27 +563,36 @@ def _insert(conn, session, text):
                   ts if ts is not None and -2 ** 63 <= ts < 2 ** 63 else None, text))
 
 
+def _line_text(line):
+    """The row text one legacy ledger line (no newline) imports as, None for a
+    blank one: kept as it was written, so a line that does not parse stays a
+    line the reader names as damage."""
+    if not line.strip():
+        return None
+    try:
+        return line.decode("utf-8")
+    except UnicodeDecodeError:
+        # the marker `tezgah_integrity._parse` names as damage, hashed over the
+        # line with its newline as the file reader hashed it, so a damage row
+        # it wrote still names the same line
+        return "\x00not utf-8 %s" % hashlib.sha1(line + b"\n").hexdigest()[:12]
+
+
 def _evidence_put(session):
-    """The import of one legacy ledger line: kept as it was written, so a line
-    that does not parse stays a line the reader names as damage."""
+    """The import of one legacy ledger line (`_line_text`)."""
     def put(conn, line):
-        if not line.strip():
-            return
-        try:
-            text = line.decode("utf-8")
-        except UnicodeDecodeError:
-            # the marker `tezgah_integrity._parse` names as damage, hashed
-            # over the line with its newline as the file reader hashed it, so
-            # a damage row it wrote still names the same line
-            text = "\x00not utf-8 %s" % hashlib.sha1(line + b"\n").hexdigest()[:12]
-        _insert(conn, session, text)
+        text = _line_text(line)
+        if text is not None:
+            _insert(conn, session, text)
     return put
 
 
 def _sync(held, path):
     """Import the legacy JSONL at `path` past the bytes already in. Without a
     file, or with nothing new in it, this is one stat (and one lookup the first
-    time a process sees it)."""
+    time a process sees it). A file whose offset sits under another device id
+    (`_recorded`) goes through the import even with nothing new, whose write
+    re-keys it."""
     try:
         st = os.stat(path)
     except OSError:
@@ -553,9 +602,7 @@ def _sync(held, path):
     if seen.get(path) == mark:
         return
     name = os.path.basename(path)
-    found = conn.execute("SELECT bytes FROM imported WHERE name = ? AND dev = ? AND ino = ?",
-                         (name, st.st_dev, st.st_ino)).fetchone()
-    if found and found[0] == st.st_size:
+    if _recorded(conn, name, st) == (st.st_size, None):
         seen[path] = mark
         return
     if conn.execute("SELECT 1 FROM import_failed WHERE name = ? AND mtime_ns = ? "