cctally 1.114.0 → 1.115.0

package/bin/_lib_journal.py

@@ -32,6 +32,8 @@ import hashlib
 import json
 from dataclasses import dataclass
 
+import _lib_account_protocol
+
 LINE_VERSION = 1
 
 SEGMENT_PREFIX = "observations-"
@@ -379,8 +381,13 @@ def effects_payload_digest(payload: dict, refs: dict | None = None) -> str:
 # fully-formed line records
 # --------------------------------------------------------------------------
 
+_WRITER_GENERATION = _lib_account_protocol.PROTOCOL_GENERATION
+_WPG = _lib_account_protocol.WRITER_GENERATION_KEY
+
+
 def make_obs(at: str, src: str, provider: str, payload: dict,
-             account: str | None = None) -> dict:
+             account: str | None = None, *,
+             writer_generation: str | None = _WRITER_GENERATION) -> dict:
     """Build a complete ``obs`` line record (raw capture; canonicalization is
     derivation-time, never baked into the stored line — spec §4.2).
 
@@ -388,20 +395,126 @@ def make_obs(at: str, src: str, provider: str, payload: dict,
     ``provider``. It is emitted ONLY when supplied, so a pre-epic / single-account
     writer that passes nothing produces a byte-identical line (and a byte-stable
     content id). When supplied it participates in the content id, so two obs that
-    differ only by account are distinct records."""
+    differ only by account are distinct records.
+
+    ``writer_generation`` (#825/#832) is the top-level ``wpg`` stamp naming the
+    writing protocol. It is part of the content-id core, so a current record and
+    a previous-release record of the same capture are distinct lines. ``None``
+    builds the previous release's exact shape (a validator re-deriving a
+    retained pre-stamp record, or a test simulating an old writer)."""
     core = {"t": "obs", "at": at, "src": src, "provider": provider, "payload": payload}
     if account is not None:
         core["account"] = account
+    if writer_generation is not None:
+        core[_WPG] = writer_generation
     return {"v": LINE_VERSION, **core, "id": content_id(core)}
 
 
-def make_op(at: str, src: str, payload: dict) -> dict:
-    """Build a complete ``op`` (operator record) line — no ``provider``."""
+def make_op(at: str, src: str, payload: dict, *,
+            writer_generation: str | None = _WRITER_GENERATION) -> dict:
+    """Build a complete ``op`` (operator record) line — no ``provider``.
+    ``writer_generation`` as in ``make_obs``."""
     core = {"t": "op", "at": at, "src": src, "payload": payload}
+    if writer_generation is not None:
+        core[_WPG] = writer_generation
     return {"v": LINE_VERSION, **core, "id": content_id(core)}
 
 
+def prepare_record(record: dict) -> dict:
+    """The single shared preparation step both append APIs apply (#825/#832
+    spec 5.1): return ``record`` carrying the current writer generation.
+
+    A content-id record (``id`` starting ``o:``) that is not yet stamped and
+    whose id verifies over its unstamped core is re-identified over the stamped
+    core, exactly as the builders compute it. Every other id — semantic event
+    ids, bootstrap ids, correction batch ids, a content id that does not verify
+    — is preserved. Idempotent: a stamped record is returned unchanged."""
+    if _lib_account_protocol.is_stamped(record):
+        return record
+    stamped = _lib_account_protocol.stamp(record)
+    rid = record.get("id")
+    if (record.get("t") in ("obs", "op") and isinstance(rid, str)
+            and rid.startswith("o:")):
+        core = {k: v for k, v in record.items() if k not in ("v", "id")}
+        if content_id(core) == rid:
+            stamped_core = {k: v for k, v in stamped.items()
+                            if k not in ("v", "id")}
+            stamped["id"] = content_id(stamped_core)
+    return stamped
+
+
+def unstamped_record(record: dict) -> dict:
+    """The previous release's shape of ``record``: the inverse of
+    ``prepare_record``. Drops the writer-generation stamp and, for a content-id
+    record whose id verifies over its stamped core, re-identifies it over the
+    unstamped core. Any other id is preserved. For fixtures and tests that
+    simulate a pre-stamp writer; never used on a production write path."""
+    if not isinstance(record, dict) or _WPG not in record:
+        return record
+    out = {k: v for k, v in record.items() if k != _WPG}
+    rid = record.get("id")
+    if (record.get("t") in ("obs", "op") and isinstance(rid, str)
+            and rid.startswith("o:")):
+        core = {k: v for k, v in record.items() if k not in ("v", "id")}
+        if content_id(core) == rid:
+            out["id"] = content_id(
+                {k: v for k, v in out.items() if k not in ("v", "id")})
+    return out
+
+
+def legacy_writer_view():
+    """A namespace exposing this module with every ``make_*`` builder wrapped by
+    ``unstamped_record``, so a fixture builder written against the module
+    produces the previous release's exact bytes (cross-references between
+    records stay consistent because every id is the unstamped one)."""
+    import types
+
+    def _wrap(fn):
+        def build(*args, **kwargs):
+            out = fn(*args, **kwargs)
+            if isinstance(out, list):
+                return [unstamped_record(item) for item in out]
+            return unstamped_record(out)
+        build.__name__ = fn.__name__
+        build.__doc__ = fn.__doc__
+        return build
+
+    def _legacy_protocol_resolution(**kwargs):
+        # The previous release's acknowledgement kind, which the current
+        # binary never emits (#825/#832 spec 5.1).
+        current = make_protocol_resolution(**kwargs)
+        return make_op(
+            at=current["at"], src=current["src"],
+            payload={**current["payload"], "kind": _PROTOCOL_RESOLUTION_KIND},
+            writer_generation=None)
+
+    attrs = {}
+    for name, value in globals().items():
+        if name.startswith("__"):
+            continue
+        if name.startswith("make_") and callable(value):
+            value = _wrap(value)
+        attrs[name] = value
+    attrs["make_protocol_resolution"] = _legacy_protocol_resolution
+    return types.SimpleNamespace(**attrs)
+
+
+def event_content_hash(record: dict) -> str:
+    """The semantic content hash of an evt line. The writer-generation stamp is
+    transport metadata, so it is stripped before hashing: a current emission
+    that differs from a retained pre-stamp event only by ``wpg`` is the same
+    event (a duplicate), never a same-revision conflict."""
+    return _sha256_canonical(_lib_account_protocol.strip_stamp(record))
+
+
+#: The previous release's exact-key repair acknowledgement. The current binary
+#: never emits it: its validator would reject the writer-generation stamp, so
+#: current acknowledgements use the versioned kind the previous release ignores
+#: as an unknown op (#825/#832 spec 5.1). Current readers accept both.
 _PROTOCOL_RESOLUTION_KIND = "journal_protocol_resolution"
+_PROTOCOL_RESOLUTION_V2_KIND = _lib_account_protocol.PROTOCOL_RESOLUTION_V2_KIND
+PROTOCOL_RESOLUTION_KINDS = frozenset((
+    _PROTOCOL_RESOLUTION_KIND, _PROTOCOL_RESOLUTION_V2_KIND))
 _PROTOCOL_RESOLUTION_SRC = "journal-repair"
 
 
@@ -441,7 +554,7 @@ def make_protocol_resolution(
         at=at,
         src=_PROTOCOL_RESOLUTION_SRC,
         payload={
-            "kind": _PROTOCOL_RESOLUTION_KIND,
+            "kind": _PROTOCOL_RESOLUTION_V2_KIND,
             "violations": refs,
             "journal_high_water": {
                 "segment": segment,
@@ -452,6 +565,17 @@ def make_protocol_resolution(
     )
 
 
+def is_protocol_resolution(record) -> bool:
+    """Whether ``record`` is a journal-repair acknowledgement of either
+    version."""
+    return (
+        isinstance(record, dict)
+        and record.get("t") == "op"
+        and isinstance(record.get("payload"), dict)
+        and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
+    )
+
+
 def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
     """Build a complete ``evt`` (derived) line record.
 
@@ -463,7 +587,8 @@ def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
     body = dict(payload)
     body["kind"] = kind
     return {"v": LINE_VERSION, "t": "evt", "id": id, "rev": rev,
-            "at": at, "src": "ingest", "payload": body}
+            "at": at, "src": "ingest", "payload": body,
+            _WPG: _WRITER_GENERATION}
 
 
 # --------------------------------------------------------------------------
@@ -501,7 +626,9 @@ def _protocol_violation(
 
 def _validate_protocol_resolution(record: dict) -> dict:
     required = {"v", "t", "id", "at", "src", "payload"}
-    if set(record) != required:
+    # #825/#832: a current record also carries the writer-generation stamp,
+    # which is part of its content-id core.
+    if set(record) not in (required, required | {_WPG}):
         raise JournalProtocolError(
             "journal protocol resolution record shape is invalid"
         )
@@ -521,6 +648,8 @@ def _validate_protocol_resolution(record: dict) -> dict:
         "src": record["src"],
         "payload": record["payload"],
     }
+    if _WPG in record:
+        core[_WPG] = record[_WPG]
     if record.get("id") != content_id(core):
         raise JournalProtocolError(
             "journal protocol resolution record id does not match its content"
@@ -535,7 +664,7 @@ def _validate_protocol_resolution(record: dict) -> dict:
         raise JournalProtocolError(
             "journal protocol resolution payload shape is invalid"
         )
-    if payload.get("kind") != _PROTOCOL_RESOLUTION_KIND:
+    if payload.get("kind") not in PROTOCOL_RESOLUTION_KINDS:
         raise JournalProtocolError(
             "journal protocol resolution kind is invalid"
         )
@@ -672,6 +801,7 @@ def make_correction_batch(
         "family": family,
         "action_count": len(cores),
         "actions_hash": actions_hash,
+        _WPG: _WRITER_GENERATION,
     }
     records = [{**marker, "phase": "begin"}]
     for seq, core in enumerate(cores):
@@ -682,6 +812,7 @@ def make_correction_batch(
                 **core,
                 "batch": batch_id,
                 "seq": seq,
+                _WPG: _WRITER_GENERATION,
             }
         )
     records.append({**marker, "phase": "commit"})
@@ -769,7 +900,7 @@ def _candidate_from_evt(record: dict, sequence: int) -> EffectiveEvent:
         )
     if not isinstance(record.get("payload"), dict):
         raise JournalProtocolError("event payload must be an object")
-    digest = _sha256_canonical(record)
+    digest = event_content_hash(record)
     return EffectiveEvent(
         event_id=event_id,
         rev=rev,
@@ -919,7 +1050,7 @@ def _fold_one(fold: SelectorFold, record, sequence: int) -> None:
     if (
         record_type == "op"
         and isinstance(record.get("payload"), dict)
-        and record["payload"].get("kind") == _PROTOCOL_RESOLUTION_KIND
+        and record["payload"].get("kind") in PROTOCOL_RESOLUTION_KINDS
     ):
         resolutions.append(
             (sequence, _validate_protocol_resolution(record))

package/bin/_lib_jsonl.py

@@ -359,7 +359,7 @@ def _iter_jsonl_entries_with_offsets(fh, path_str: str):
 
 
 _CODEX_FILENAME_UUID_RE = re.compile(
-    r"rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-([0-9a-fA-F-]{36})\.jsonl$"
+    r"rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-([0-9a-fA-F-]{36})\.jsonl(?:\.zst)?$"
 )
 
 
@@ -883,7 +883,7 @@ def _reject_nonfinite_json_constant(value: str) -> None:
 
 
 def _codex_generation_transition_between(
-    path_str: str, after_offset: int, before_offset: int,
+    path_str: str, after_offset: int, before_offset: int, *, open_stream=None,
 ) -> bool:
     """Return whether a producer lifecycle boundary exists in this byte gap.
 
@@ -892,12 +892,13 @@ def _codex_generation_transition_between(
     and is never consumed. This reconstructs the one bit of parser state a
     metadata-only or budgeted pass can leave between the last accounting row
     and the next token event without adding another cache schema column.
-    Malformed or unreadable evidence fails closed.
+    Malformed or unreadable evidence fails closed. A container walk supplies
+    ``open_stream`` so all gap offsets address decoded bytes.
     """
     if after_offset < 0 or before_offset <= after_offset:
         return False
     try:
-        with open(path_str, "rb") as fh:
+        with (open_stream() if open_stream is not None else open(path_str, "rb")) as fh:
             fh.seek(after_offset)
             fh.readline()  # skip the retained accounting record itself
             while fh.tell() < before_offset:

package/bin/_lib_legacy_spool.py

@@ -0,0 +1,126 @@
+"""Pure legacy statusline spool baseline (#825/#832, spec sections 5.5 and 7).
+
+After the account-bound cutover the previous release's statusline spool
+(``statusline-candidates/``, ``statusline-selected.json``, the authoritative
+tombstones and the freshness markers) is never read as authority. It is only
+compared with a baseline captured inside the locked cutover establishment and
+anchored in the journal: the cutover op records the baseline digest, and every
+later ``legacy_spool_activity`` op records the digest the baseline advances to.
+A change against the baseline is evidence that a legacy process wrote after the
+cutover, which quarantines automatic outcomes (spec 5.6 item 1(b)).
+
+This module owns the baseline's value shape, its digest, its on-disk document
+codec and the comparison. It is pure: the filesystem capture lives in
+``_cctally_journal.capture_legacy_spool_state``. File content is digested,
+never interpreted.
+"""
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import json
+import re
+
+DOCUMENT_SCHEMA_VERSION = 1
+
+_SHA_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+
+
+def _canonical(value) -> bytes:
+    return json.dumps(value, separators=(",", ":"), sort_keys=True,
+                      ensure_ascii=False).encode("utf-8")
+
+
+def content_digest(data: bytes) -> str:
+    return "sha256:" + hashlib.sha256(data).hexdigest()
+
+
+@dataclasses.dataclass(frozen=True)
+class LegacySpoolState:
+    """The observed legacy spool: one ``(relative path, size, mtime_ns,
+    sha256 of content)`` entry per regular file, sorted by path."""
+
+    entries: tuple[tuple[str, int, int, str], ...]
+
+    def digest(self) -> str:
+        return content_digest(_canonical([list(e) for e in self.entries]))
+
+
+def make_state(entries) -> LegacySpoolState:
+    """Normalize ``entries`` into a sorted, validated state."""
+    normalized = []
+    seen = set()
+    for entry in entries:
+        path, size, mtime_ns, sha = entry
+        if not isinstance(path, str) or not path or path in seen:
+            raise ValueError("legacy spool entry path is invalid")
+        if type(size) is not int or size < 0:
+            raise ValueError("legacy spool entry size is invalid")
+        if type(mtime_ns) is not int:
+            raise ValueError("legacy spool entry mtime is invalid")
+        if not isinstance(sha, str) or not _SHA_RE.match(sha):
+            raise ValueError("legacy spool entry digest is invalid")
+        seen.add(path)
+        normalized.append((path, size, mtime_ns, sha))
+    return LegacySpoolState(entries=tuple(sorted(normalized)))
+
+
+def diff(baseline: LegacySpoolState,
+         current: LegacySpoolState) -> tuple[str, ...]:
+    """The relative paths that were added, removed or changed (any of size,
+    modification time or content), sorted."""
+    before = {e[0]: e[1:] for e in baseline.entries}
+    after = {e[0]: e[1:] for e in current.entries}
+    return tuple(sorted(
+        path for path in set(before) | set(after)
+        if before.get(path) != after.get(path)
+    ))
+
+
+def change_evidence(changed, current: LegacySpoolState) -> list[dict]:
+    """Per changed path, the observed after-state (or its absence). Digests
+    and metadata only, never a legacy value."""
+    after = {e[0]: e for e in current.entries}
+    out = []
+    for path in sorted(changed):
+        entry = after.get(path)
+        if entry is None:
+            out.append({"path": path, "absent": True})
+        else:
+            out.append({"path": path, "size": entry[1], "mtimeNs": entry[2],
+                        "sha256": entry[3]})
+    return out
+
+
+def to_document(state: LegacySpoolState) -> dict:
+    return {
+        "schemaVersion": DOCUMENT_SCHEMA_VERSION,
+        "digest": state.digest(),
+        "entries": [
+            {"path": p, "size": s, "mtimeNs": m, "sha256": h}
+            for (p, s, m, h) in state.entries
+        ],
+    }
+
+
+def from_document(doc) -> LegacySpoolState:
+    """Decode a baseline document. Raises ``ValueError`` on any shape failure
+    or when the recorded digest does not match the entries."""
+    if not isinstance(doc, dict) or doc.get("schemaVersion") != (
+            DOCUMENT_SCHEMA_VERSION):
+        raise ValueError("legacy spool baseline document shape is invalid")
+    entries = doc.get("entries")
+    if not isinstance(entries, list):
+        raise ValueError("legacy spool baseline entries are invalid")
+    try:
+        state = make_state(
+            (e["path"], e["size"], e["mtimeNs"], e["sha256"])
+            for e in entries if isinstance(e, dict)
+        )
+    except (KeyError, TypeError) as exc:
+        raise ValueError("legacy spool baseline entry is invalid") from exc
+    if len(state.entries) != len(entries):
+        raise ValueError("legacy spool baseline entry is invalid")
+    if doc.get("digest") != state.digest():
+        raise ValueError("legacy spool baseline digest does not match")
+    return state

package/bin/_lib_legacy_writer_evidence.py

@@ -0,0 +1,136 @@
+"""Pure legacy-writer evidence (#825/#832, spec section 8, case F9).
+
+After the account-bound cutover, records an older cctally process appends are
+excluded from current accounting. `doctor`'s `journal.legacy_writer` leg warns
+while that is still happening, and clears after seven quiet days (operator
+Q4). The seven-day clock needs evidence of WHEN the excluded population was
+first observed, and neither the disposable index (rebuilt) nor a record's `at`
+(event time, which a backdated legacy event fakes) provides it. So one small
+durable sidecar records:
+
+* the cutover identity;
+* the excluded population: its count and an ORDER-INDEPENDENT digest over the
+  physical coordinates of every excluded complete record;
+* `firstObservedAt`: the wall-clock time a current scan first observed THAT
+  population.
+
+The trigger is the population of physical coordinates, never a record's `at`
+or the maximum coordinate alone, so a record added to an earlier segment, a
+new bootstrap segment, or a backdated event all count as new evidence. A scan
+that finds the same population leaves the evidence untouched, so a rebuild
+over unchanged evidence does not restart the age. A missing sidecar is
+recreated with the current time, which can only lengthen a warning. The
+sidecar is diagnostic only: it never decides admission.
+
+Pure: no I/O, no clock (callers pass `now`).
+"""
+from __future__ import annotations
+
+import dataclasses
+import datetime as dt
+import re
+
+SCHEMA_VERSION = 1
+QUIET_DAYS = 7
+REMEDY = ("an older cctally process (for example a dashboard started before "
+          "the upgrade) is still writing; restart it")
+
+_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+
+
+@dataclasses.dataclass(frozen=True)
+class Evidence:
+    cutover_id: str
+    excluded_count: int
+    coordinate_digest: str
+    first_observed_at: str  # ISO-8601 UTC
+
+
+def _parse_instant(value):
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return None
+    return parsed.astimezone(dt.timezone.utc)
+
+
+def _iso(now: dt.datetime) -> str:
+    return now.astimezone(dt.timezone.utc).isoformat(timespec="seconds")
+
+
+def to_document(evidence: Evidence) -> dict:
+    return {
+        "schemaVersion": SCHEMA_VERSION,
+        "cutoverId": evidence.cutover_id,
+        "excludedCount": evidence.excluded_count,
+        "coordinateDigest": evidence.coordinate_digest,
+        "firstObservedAt": evidence.first_observed_at,
+    }
+
+
+def from_document(doc) -> Evidence | None:
+    """The evidence a sidecar document holds, or None when it is not a valid
+    document (treated exactly like a missing sidecar)."""
+    if not isinstance(doc, dict) or doc.get("schemaVersion") != SCHEMA_VERSION:
+        return None
+    cutover_id = doc.get("cutoverId")
+    count = doc.get("excludedCount")
+    digest = doc.get("coordinateDigest")
+    first = doc.get("firstObservedAt")
+    if (not isinstance(cutover_id, str) or not cutover_id
+            or type(count) is not int or count < 0
+            or not isinstance(digest, str) or not _DIGEST_RE.match(digest)
+            or _parse_instant(first) is None):
+        return None
+    return Evidence(cutover_id, count, digest, first)
+
+
+def observe(existing: Evidence | None, *, cutover_id: str | None,
+            excluded_count: int, coordinate_digest: str | None,
+            now: dt.datetime) -> Evidence | None:
+    """The update rule. Returns the evidence a scan should leave behind:
+
+    * no cutover, or no excluded record at all: ``existing`` unchanged (None
+      when there was none) — nothing to warn about;
+    * the same cutover, count and digest: ``existing`` unchanged, so the age
+      keeps running;
+    * anything else (a new population, or no valid sidecar): new evidence
+      first observed ``now``.
+    """
+    if cutover_id is None or excluded_count <= 0 or coordinate_digest is None:
+        return existing
+    if (existing is not None
+            and existing.cutover_id == cutover_id
+            and existing.excluded_count == excluded_count
+            and existing.coordinate_digest == coordinate_digest):
+        return existing
+    return Evidence(cutover_id, int(excluded_count), coordinate_digest,
+                    _iso(now))
+
+
+@dataclasses.dataclass(frozen=True)
+class Verdict:
+    severity: str          # "ok" | "warn"
+    state: str             # "none" | "recent" | "quiet"
+    excluded_count: int
+    last_seen_at: str | None
+    age_seconds: int | None
+
+
+def classify(evidence: Evidence | None, *, excluded_count: int,
+             now: dt.datetime) -> Verdict:
+    """The leg's verdict: WARN while the latest excluded population was first
+    observed under `QUIET_DAYS` ago, OK with a historical note afterwards,
+    OK with nothing to report when nothing was ever excluded."""
+    if excluded_count <= 0 or evidence is None:
+        return Verdict("ok", "none", max(0, excluded_count), None, None)
+    first = _parse_instant(evidence.first_observed_at)
+    age = max(0, int((now.astimezone(dt.timezone.utc) - first).total_seconds()))
+    recent = age < QUIET_DAYS * 86400
+    return Verdict("warn" if recent else "ok",
+                   "recent" if recent else "quiet",
+                   excluded_count, evidence.first_observed_at, age)