cctally 1.79.2 → 1.80.0

package/bin/_cctally_sync_week.py

@@ -43,15 +43,49 @@ from _cctally_core import (
     format_local_iso,
     make_week_ref,
     get_latest_usage_for_week,
+    now_utc_iso,
 )
 
 
-def cmd_sync_week(args: argparse.Namespace) -> int:
+def cmd_sync_week(
+    args: argparse.Namespace, *, conn=None, as_of: "str | None" = None,
+    journal: "tuple | None" = None,
+) -> int:
+    """Compute + persist the week's cost snapshot.
+
+    Transaction-neutral / capture-time-pure seam (DB journal redesign §5.2.3):
+    ``conn`` runs the cost-snapshot insert on the caller's connection (no
+    internal open/commit/close), and ``as_of`` (ISO-Z) stamps the snapshot's
+    ``captured_at_utc``. Both defaults keep the legacy own-connection,
+    wall-clock behavior — so `record`/`maybe_record_milestone`'s existing
+    bare `cmd_sync_week(ns)` calls are byte-identical.
+
+    Design A (DB journal redesign §5.3, Model-A ``weekly_cost_snapshot``):
+    ``journal=(ctx, id_base)`` routes the ``insert_cost_snapshot`` THROUGH
+    ``emit_model_a`` so the computed cost rides a journaled evt
+    (``wcs:<id_base>:<week>``) and replay reads it back verbatim rather than
+    recomputing from the (pruned) provider JSONL. Threaded by the
+    milestone-cost-sync (id_base = the obs line id) and the sync-week op fold
+    (id_base = the op line id). Default ``None`` keeps the bare insert.
+
+    6f writer reroute (Appendix A): the pure-CLI / bare-call path (``conn`` is
+    None AND ``journal`` is None — the ``sync-week`` subcommand entry and the
+    ``report --sync-current`` lazy-sync) no longer computes + inserts on its own
+    connection. It appends a ``sync_week`` ``op`` line and runs an AUTHORITATIVE
+    ingest — ``_pipeline_sync_week`` re-derives args from the op payload and
+    re-enters THIS function with ``conn``+``journal`` set (the inline body below),
+    which computes the cost on the cycle connection and journals the
+    ``weekly_cost_snapshots`` row — then reads that row back for its output. The
+    passed-conn / journal callers stay inline and byte-identical."""
     c = _cctally()
+    if conn is None and journal is None:
+        return _cmd_sync_week_via_ingest(c, args)
     config = c.load_config()
     week_start_name = get_week_start_name(config, args.week_start_name)
 
-    conn = open_db()
+    own_conn = conn is None
+    if own_conn:
+        conn = open_db()
     try:
         selection = c.pick_week_selection(
             conn,
@@ -83,6 +117,9 @@ def cmd_sync_week(args: argparse.Namespace) -> int:
             cost_usd=result.cost_usd,
             mode=args.mode,
             project=args.project,
+            commit=own_conn,
+            as_of=as_of,
+            journal=journal,
         )
 
         week_ref = make_week_ref(
@@ -110,6 +147,106 @@ def cmd_sync_week(args: argparse.Namespace) -> int:
             "dollarsPerPercent": round(dollars_per_percent, 9) if dollars_per_percent is not None else None,
         }
 
+        if args.json:
+            print(json.dumps(c.stamp_schema_version(payload), indent=2))
+        elif not args.quiet:
+            print(
+                f"Synced week {payload['weekStartDate']}..{payload['weekEndDate']} "
+                f"=> ${payload['costUSD']:.6f}"
+            )
+            if weekly_percent is not None:
+                print(f"Latest weekly usage: {weekly_percent:.2f}%")
+            if dollars_per_percent is not None:
+                print(f"$ per 1% usage: ${dollars_per_percent:.6f}")
+        return 0
+    finally:
+        if own_conn:
+            conn.close()
+
+
+def _cmd_sync_week_via_ingest(c, args: argparse.Namespace) -> int:
+    """CLI / bare-call sync-week (6f writer reroute): append a ``sync_week`` op
+    + AUTHORITATIVE ingest, then read the journaled ``weekly_cost_snapshots`` row
+    back for output. The op carries the raw ``sync-week`` args; the
+    ``_pipeline_sync_week`` hook re-enters ``cmd_sync_week`` with the cycle
+    connection to compute + journal the cost (Model-A ``wcs:<op id>:<week>``).
+
+    The command observes its own write synchronously (authoritative ingest, then
+    a fresh read-back). Exit codes stay byte-identical: a genuine compute/insert
+    failure propagates out of the authoritative ingest exactly as the old inline
+    path let ``compute_week_cost``/``insert_cost_snapshot`` raise, and the CLI
+    dispatch maps it to the same exit code."""
+    import _cctally_journal as _jr
+    import _lib_journal as _lj
+
+    op = _lj.make_op(
+        at=now_utc_iso(),
+        src="sync-week",
+        payload={
+            "kind": "sync_week",
+            "week_start": args.week_start,
+            "week_end": args.week_end,
+            "week_start_name": args.week_start_name,
+            "mode": args.mode,
+            "offline": args.offline,
+            "project": args.project,
+        },
+    )
+    _jr.append_record(op)
+    res = _jr.run_stats_ingest(mode="authoritative")
+    # authoritative blocks up to the ingest-lock timeout; on the rare timeout
+    # (busy lock) it returns ran=False without consuming the op — retry once so
+    # the CLI observes its own write.
+    if not res.ran:
+        res = _jr.run_stats_ingest(mode="authoritative")
+
+    conn = open_db()
+    try:
+        row = conn.execute(
+            "SELECT id, week_start_date, week_end_date, week_start_at, "
+            "       week_end_at, range_start_iso, range_end_iso, cost_usd, mode "
+            "  FROM weekly_cost_snapshots "
+            " WHERE journal_id LIKE ? "
+            " ORDER BY id DESC LIMIT 1",
+            (f"wcs:{op['id']}:%",),
+        ).fetchone()
+        if row is None:
+            # The op did not journal a cost row (only reachable if the ingest
+            # never ran — a sustained ingest-lock timeout). Surface it rather
+            # than printing a phantom success.
+            import sys
+            print(
+                "sync-week: cost ingest did not run (ingest lock busy); retry",
+                file=sys.stderr,
+            )
+            return 3
+
+        (row_id, wsd, wed, wsa, wea, r_start, r_end, cost_usd, _mode) = row
+        week_ref = make_week_ref(
+            week_start_date=wsd,
+            week_end_date=wed,
+            week_start_at=wsa,
+            week_end_at=wea,
+        )
+        usage_row = get_latest_usage_for_week(conn, week_ref)
+        weekly_percent = float(usage_row["weekly_percent"]) if usage_row else None
+        dollars_per_percent = (
+            cost_usd / weekly_percent if weekly_percent and weekly_percent > 0 else None
+        )
+
+        payload = {
+            "id": row_id,
+            "weekStartDate": wsd,
+            "weekEndDate": wed,
+            "weekStartAt": wsa,
+            "weekEndAt": wea,
+            "rangeStartIso": r_start,
+            "rangeEndIso": r_end,
+            "costUSD": round(cost_usd, 9),
+            "weeklyPercent": weekly_percent,
+            "dollarsPerPercent": round(dollars_per_percent, 9) if dollars_per_percent is not None else None,
+        }
+
         if args.json:
             print(json.dumps(c.stamp_schema_version(payload), indent=2))
         elif not args.quiet:

package/bin/_lib_doctor.py

@@ -241,6 +241,37 @@ class DoctorState:
     # check (WARN on beta+brew, else OK). Defaulted (placed last) so existing
     # constructors stay valid and default to the stable posture.
     update_channel: str = "stable"
+    # DB journal redesign §9: the append-only journal legs. All gathered by
+    # `doctor_gather_state` (read-only), defaulted (tail) so existing
+    # constructors stay valid and a pre-cutover install (no journal dir) reads
+    # the always-OK "no journal" posture.
+    #   * journal_present — the journal/ dir exists (a cut-over install has one;
+    #     a legacy pre-cutover install does NOT — that is INFO, never FAIL).
+    #   * journal_appendable — the dir is writable (os.access W_OK); None when
+    #     absent or the probe errored.
+    #   * journal_segment_count — number of segments (bootstrap + monthly).
+    #   * journal_malformed_count / journal_torn_tail_count — mid-file malformed
+    #     lines (external damage → WARN) and torn final lines (a known crash
+    #     artifact healed by the next append → INFO). None = not scanned (the
+    #     scan is `deep`-gated, like the quick_check legs, so the dashboard's
+    #     per-rebuild gather never reads the whole journal).
+    #   * journal_cursor_lag_bytes — unconsumed bytes between the stats index
+    #     cursor and the journal high-water. None when there is no journal /
+    #     cursor / the DB could not be read.
+    #   * journal_hw_segment / journal_cursor_segment — the high-water segment
+    #     and the cursor's segment, for the verbose detail.
+    #   * journal_heal_incidents — most-recent-first list of the auto-heal
+    #     artifacts (quarantine/ dirs + logs/<db>-corruption-forensics-*.json);
+    #     each dict carries {kind, name, age_s}. None = the dirs were unreadable.
+    journal_present: bool = False
+    journal_appendable: Optional[bool] = None
+    journal_segment_count: int = 0
+    journal_malformed_count: Optional[int] = None
+    journal_torn_tail_count: Optional[int] = None
+    journal_cursor_lag_bytes: Optional[int] = None
+    journal_hw_segment: Optional[str] = None
+    journal_cursor_segment: Optional[str] = None
+    journal_heal_incidents: Optional[list] = None
 
 
 @dataclasses.dataclass(frozen=True)
@@ -731,29 +762,58 @@ def _check_db_migrations_pending(s: DoctorState) -> CheckResult:
 
 
 def _check_db_version_ahead(s: DoctorState) -> CheckResult:
-    """FAIL/WARN when a DB's user_version exceeds the running binary's
-    registry head (issue #145). stats.db ahead bricks commands (FAIL);
-    cache.db ahead auto-heals on the next open (WARN). doctor reads raw
-    user_version (no dispatcher), so it can report without healing/bricking.
+    """Classify each DB's ``user_version`` versus what this binary expects.
+
+    stats.db follows the EPOCH model (DB journal redesign §7.1): it is a
+    DISPOSABLE index stamped at ``STATS_INDEX_EPOCH`` (injected as ``epoch``
+    into ``stats_db_status`` by the gather layer), NOT a versioned migration
+    target. Classification:
+      * ``uv == epoch`` (a cut-over install)      → HEALTHY (steady state)
+      * ``uv <= legacy_head`` (pre-cutover, ≤13)  → HEALTHY (cuts over on open)
+      * ``uv > legacy_head`` AND ``!= epoch``      → §7.1 index MISMATCH: WARN.
+        It self-heals by journal REBUILD on the next open (never bricks, unlike
+        the retired #145 version-ahead FAIL), so the remediation points at
+        `db rebuild --db stats`, NOT the retired `db recover --db stats`.
+
+    cache.db is unchanged (issue #145): a ``user_version`` past the cache
+    registry head auto-heals on the next open → WARN. doctor reads raw
+    ``user_version`` (no dispatcher), so it reports without healing/bricking.
     """
-    def _eval(status):
+    def _eval_stats(status):
+        if not status:
+            return None
+        uv = status.get("user_version", 0) or 0
+        legacy_head = status.get("registry_size", 0) or 0  # frozen stats head (13)
+        epoch = status.get("epoch")
+        if epoch is None:
+            # Fallback kept in lockstep with _cctally_core.STATS_INDEX_EPOCH; the
+            # gather layer injects the real constant, so this only guards a hand-
+            # built DoctorState that omitted it.
+            epoch = 1000
+        mismatch = uv > legacy_head and uv != epoch
+        return {"user_version": uv, "legacy_head": legacy_head, "epoch": epoch,
+                "mismatch": mismatch}
+
+    def _eval_cache(status):
         if not status:
             return None
         uv = status.get("user_version", 0) or 0
         rs = status.get("registry_size", 0) or 0
         return {"user_version": uv, "registry_size": rs, "ahead": uv > rs}
 
-    stats = _eval(s.stats_db_status)
-    cache = _eval(s.cache_db_status)
+    stats = _eval_stats(s.stats_db_status)
+    cache = _eval_cache(s.cache_db_status)
     details = {"stats.db": stats, "cache.db": cache}
-    stats_ahead = bool(stats and stats["ahead"])
+    stats_mismatch = bool(stats and stats["mismatch"])
     cache_ahead = bool(cache and cache["ahead"])
 
-    if stats_ahead:
+    if stats_mismatch:
         return CheckResult(
-            id="db.version_ahead", title="Version ahead", severity="fail",
-            summary=f"stats.db ahead (v{stats['user_version']} > known v{stats['registry_size']})",
-            remediation="Run `cctally db recover --db stats --yes` (or restore from backup)",
+            id="db.version_ahead", title="Version ahead", severity="warn",
+            summary=(f"stats.db index mismatch (v{stats['user_version']} ≠ epoch "
+                     f"v{stats['epoch']}) — rebuilds from journal"),
+            remediation=("Auto-heals by journal rebuild on next command; if it "
+                         "persists, run `cctally db rebuild --db stats`"),
             details=details,
         )
     if cache_ahead:
@@ -1858,6 +1918,137 @@ def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
     )
 
 
+# ── DB journal redesign §9 — append-only journal legs ────────────────────
+# A monthly segment is MB-scale (§4.5), so a multi-MB unconsumed cursor gap
+# means no ingest cycle has run for a long stretch. An auto-heal incident within
+# a week is worth surfacing loudly (the DB corrupted and self-healed).
+_JOURNAL_CURSOR_LAG_WARN_BYTES = 4 * 1024 * 1024
+_JOURNAL_HEAL_RECENT_SECONDS = 7 * 24 * 3600
+
+
+def _check_journal_presence(s: DoctorState) -> CheckResult:
+    """The journal directory exists and is appendable. A pre-cutover (legacy)
+    install has NO journal yet — that is INFO (OK), never a FAIL."""
+    if not s.journal_present:
+        return CheckResult(
+            id="journal.presence", title="Journal", severity="ok",
+            summary="no journal (pre-cutover install)", remediation=None,
+            details={"present": False},
+        )
+    details = {"present": True, "appendable": s.journal_appendable,
+               "segments": s.journal_segment_count}
+    if s.journal_appendable is False:
+        return CheckResult(
+            id="journal.presence", title="Journal", severity="warn",
+            summary="journal directory not writable",
+            remediation="Fix permissions on ~/.local/share/cctally/journal/",
+            details=details,
+        )
+    return CheckResult(
+        id="journal.presence", title="Journal", severity="ok",
+        summary=f"{s.journal_segment_count} segment(s), writable",
+        remediation=None, details=details,
+    )
+
+
+def _check_journal_integrity(s: DoctorState) -> CheckResult:
+    """Mid-file malformed lines are external damage (WARN); a torn final line is
+    a known crash artifact healed by the next append (INFO). The scan is
+    deep-gated — a None count means "not scanned", always OK."""
+    if not s.journal_present:
+        return CheckResult(
+            id="journal.integrity", title="Journal integrity", severity="ok",
+            summary="no journal", remediation=None, details={"present": False},
+        )
+    if s.journal_malformed_count is None:
+        return CheckResult(
+            id="journal.integrity", title="Journal integrity", severity="ok",
+            summary="not scanned", remediation=None, details={"scanned": False},
+        )
+    torn = s.journal_torn_tail_count or 0
+    details = {"malformed": s.journal_malformed_count, "torn_tail": torn}
+    if s.journal_malformed_count > 0:
+        return CheckResult(
+            id="journal.integrity", title="Journal integrity", severity="warn",
+            summary=f"{s.journal_malformed_count} malformed line(s)",
+            remediation=("External damage to a journal segment — inspect "
+                         "~/.local/share/cctally/journal/ (every other line stays "
+                         "parseable; the ingester skips + counts the bad ones)"),
+            details=details,
+        )
+    if torn > 0:
+        return CheckResult(
+            id="journal.integrity", title="Journal integrity", severity="ok",
+            summary=f"{torn} torn tail (heals on next append)",
+            remediation=None, details=details,
+        )
+    return CheckResult(
+        id="journal.integrity", title="Journal integrity", severity="ok",
+        summary="no malformed lines", remediation=None, details=details,
+    )
+
+
+def _check_journal_index_freshness(s: DoctorState) -> CheckResult:
+    """The stats index cursor vs. the journal high-water. A large unconsumed gap
+    → WARN; a small gap or caught-up cursor → OK (with the gap shown)."""
+    if not s.journal_present or s.journal_cursor_lag_bytes is None:
+        return CheckResult(
+            id="journal.index_freshness", title="Journal index", severity="ok",
+            summary="no cursor yet", remediation=None,
+            details={"lag_bytes": None},
+        )
+    lag = s.journal_cursor_lag_bytes
+    details = {"lag_bytes": lag, "hw_segment": s.journal_hw_segment,
+               "cursor_segment": s.journal_cursor_segment,
+               "warn_bytes": _JOURNAL_CURSOR_LAG_WARN_BYTES}
+    if lag == 0:
+        return CheckResult(
+            id="journal.index_freshness", title="Journal index", severity="ok",
+            summary="index caught up", remediation=None, details=details,
+        )
+    if lag > _JOURNAL_CURSOR_LAG_WARN_BYTES:
+        return CheckResult(
+            id="journal.index_freshness", title="Journal index", severity="warn",
+            summary=f"index {lag:,} bytes behind journal",
+            remediation=("Run any cctally command (or `cctally db rebuild --db "
+                         "stats`) — the ingester consumes the backlog"),
+            details=details,
+        )
+    return CheckResult(
+        id="journal.index_freshness", title="Journal index", severity="ok",
+        summary=f"index {lag:,} bytes behind (within threshold)",
+        remediation=None, details=details,
+    )
+
+
+def _check_journal_auto_heal(s: DoctorState) -> CheckResult:
+    """The most recent auto-heal incident (quarantine dir + forensics bundle).
+    INFO listing the latest; WARN when it fired within the last 7 days."""
+    incidents = s.journal_heal_incidents
+    if not incidents:
+        return CheckResult(
+            id="journal.auto_heal", title="Auto-heal", severity="ok",
+            summary="no auto-heal incidents", remediation=None,
+            details={"incidents": 0},
+        )
+    latest = incidents[0]
+    age_s = latest.get("age_s")
+    name = latest.get("name", "?")
+    details = {"incidents": len(incidents), "latest": latest}
+    if age_s is not None and age_s <= _JOURNAL_HEAL_RECENT_SECONDS:
+        return CheckResult(
+            id="journal.auto_heal", title="Auto-heal", severity="warn",
+            summary=f"auto-heal fired recently ({name}, {age_s // 86400}d ago)",
+            remediation=("A DB corrupted and self-healed — inspect the forensics "
+                         "bundle in ~/.local/share/cctally/logs/"),
+            details=details,
+        )
+    return CheckResult(
+        id="journal.auto_heal", title="Auto-heal", severity="ok",
+        summary=f"last incident {name}", remediation=None, details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1896,6 +2087,12 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.reclaimable", "_check_db_reclaimable"),
         ("db.conversations_reclaimable", "_check_db_conversations_reclaimable"),
     )),
+    ("journal", "Journal", (
+        ("journal.presence", "_check_journal_presence"),
+        ("journal.integrity", "_check_journal_integrity"),
+        ("journal.index_freshness", "_check_journal_index_freshness"),
+        ("journal.auto_heal", "_check_journal_auto_heal"),
+    )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),
         ("data.statusline_pipeline", "_check_statusline_pipeline"),

package/bin/_lib_journal.py

@@ -0,0 +1,188 @@
+"""Pure journal kernel — line codec, identity, segment naming/order, tail scan.
+
+The durable-truth journal (design spec
+docs/superpowers/specs/2026-07-22-db-journal-redesign-design.md §4) stores one
+compact JSON object per line in monthly segments. This module owns everything
+about that format that is *pure*: encoding/decoding a line, deriving the stable
+`id` for every line class, naming and canonically ordering segments, and
+scanning a file's tail for the last complete line (torn-tail repair support).
+
+I/O-free by construction — stdlib only (`json`, `hashlib`, `datetime`), no
+imports from `_cctally_*`, no filesystem or lock access. The append/ingest I/O
+lives in `bin/_cctally_journal.py`; the durability discipline that consumes
+`valid_tail_offset`/`journal_high_water` lives there.
+
+Line format (spec §4.2), additive-evolution — readers tolerate unknown keys
+and unknown `t` values:
+
+    {"v":1,"t":"obs","id":"o:…","at":"…Z","src":"…","provider":"…","payload":{…}}
+    {"v":1,"t":"op","id":"o:…","at":"…Z","src":"record-credit","payload":{…}}
+    {"v":1,"t":"evt","id":"<natural-key>","rev":0,"at":"…Z","src":"ingest","payload":{"kind":…}}
+
+- `id`: obs/op carry a content digest over (t, at, src[, provider], payload);
+  bootstrap-exported lines use `b:<table>:<rowid>`; evt lines carry their target
+  table's full natural key with logical-id FK refs (spec §4.2 FK rule).
+- `rev`: evt revision, default 0; the fold keeps the highest rev per id
+  (reserved for the deferred correction subsystem, spec §5.5).
+"""
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import json
+
+LINE_VERSION = 1
+
+SEGMENT_PREFIX = "observations-"
+BOOTSTRAP_PREFIX = "bootstrap-"
+
+
+# --------------------------------------------------------------------------
+# line codec
+# --------------------------------------------------------------------------
+
+def _canonical_json(obj: dict) -> str:
+    """Deterministic compact JSON: sorted keys, no separator whitespace,
+    non-ASCII preserved literally. The single serialization shape used for
+    both the on-disk line and the content digest, so an id recomputed from a
+    decoded line matches the id computed at write time."""
+    return json.dumps(obj, separators=(",", ":"), sort_keys=True, ensure_ascii=False)
+
+
+def encode_line(record: dict) -> bytes:
+    """Canonical compact JSON for ``record`` plus a trailing ``\\n`` (UTF-8)."""
+    return _canonical_json(record).encode("utf-8") + b"\n"
+
+
+def decode_line(raw: bytes) -> dict | None:
+    """Decode one journal line. Returns the dict, or ``None`` on ANY parse or
+    shape failure — the line must be a JSON object carrying a string ``t``.
+
+    ``None`` is how the ingester distinguishes a malformed line (skip + count,
+    spec §4.4) from a real record; it never raises on bad input."""
+    try:
+        obj = json.loads(raw)
+    except (ValueError, TypeError):
+        return None
+    if not isinstance(obj, dict):
+        return None
+    if not isinstance(obj.get("t"), str):
+        return None
+    return obj
+
+
+# --------------------------------------------------------------------------
+# identity
+# --------------------------------------------------------------------------
+
+def content_id(record_sans_id: dict) -> str:
+    """Content digest id for an obs/op line: ``"o:" + sha256(canonical)[:16]``.
+
+    ``record_sans_id`` is the identity-bearing subset — for obs/op that is
+    ``{t, at, src[, provider], payload}`` (no ``v``, no ``id``). Stable under
+    key insertion order (canonical JSON sorts keys), so re-deriving the id from
+    a decoded line reproduces it (spec §4.2)."""
+    digest = hashlib.sha256(_canonical_json(record_sans_id).encode("utf-8")).hexdigest()
+    return "o:" + digest[:16]
+
+
+def bootstrap_id(table: str, rowid: int) -> str:
+    """Stable id for a row exported at cutover: ``b:<table>:<legacy rowid>``.
+    Stable across cutover re-runs, which is what makes double-fold idempotent
+    (spec §8)."""
+    return f"b:{table}:{rowid}"
+
+
+def evt_id(kind: str, *parts: object) -> str:
+    """Natural-key id for an evt line: ``"<kind>:" + ":".join(str(p) …)``.
+
+    Each caller passes its target table's full UNIQUE-constraint components,
+    with any DB-assigned integer FK replaced by the *logical id* of the
+    referenced record (spec §4.2). e.g.
+    ``evt_id("pm", week_start_at, reset_segment_logical_id, pct)``."""
+    return f"{kind}:" + ":".join(str(p) for p in parts)
+
+
+# --------------------------------------------------------------------------
+# fully-formed line records
+# --------------------------------------------------------------------------
+
+def make_obs(at: str, src: str, provider: str, payload: dict) -> dict:
+    """Build a complete ``obs`` line record (raw capture; canonicalization is
+    derivation-time, never baked into the stored line — spec §4.2)."""
+    core = {"t": "obs", "at": at, "src": src, "provider": provider, "payload": payload}
+    return {"v": LINE_VERSION, **core, "id": content_id(core)}
+
+
+def make_op(at: str, src: str, payload: dict) -> dict:
+    """Build a complete ``op`` (operator record) line — no ``provider``."""
+    core = {"t": "op", "at": at, "src": src, "payload": payload}
+    return {"v": LINE_VERSION, **core, "id": content_id(core)}
+
+
+def make_evt(kind: str, id: str, at: str, payload: dict, rev: int = 0) -> dict:
+    """Build a complete ``evt`` (derived) line record.
+
+    The ``id`` is the caller-built natural key (see ``evt_id``); ``kind`` is the
+    fold-dispatch family written into ``payload["kind"]`` (spec §5.3). The
+    caller's ``payload`` dict is not mutated. ``src`` is always ``"ingest"`` —
+    evt lines exist only because the ingester derived them."""
+    body = dict(payload)
+    body["kind"] = kind
+    return {"v": LINE_VERSION, "t": "evt", "id": id, "rev": rev,
+            "at": at, "src": "ingest", "payload": body}
+
+
+# --------------------------------------------------------------------------
+# segment naming + canonical order
+# --------------------------------------------------------------------------
+
+def segment_name(now_utc: dt.datetime) -> str:
+    """``observations-YYYY-MM.jsonl`` for the UTC calendar month of ``now_utc``.
+
+    A tz-aware datetime is converted to UTC first (spec §4.1: segments are cut
+    by the UTC month of the append); a naive datetime is treated as UTC."""
+    if now_utc.tzinfo is not None:
+        now_utc = now_utc.astimezone(dt.timezone.utc)
+    return f"{SEGMENT_PREFIX}{now_utc.year:04d}-{now_utc.month:02d}.jsonl"
+
+
+def segment_sort_key(name: str) -> tuple:
+    """Canonical segment order key (spec §4.1): bootstrap segments first, then
+    observation segments, each class lexicographic by name. Anything else sorts
+    last so a stray file can never wedge before real segments."""
+    if name.startswith(BOOTSTRAP_PREFIX):
+        return (0, name)
+    if name.startswith(SEGMENT_PREFIX):
+        return (1, name)
+    return (2, name)
+
+
+# --------------------------------------------------------------------------
+# torn-tail scan
+# --------------------------------------------------------------------------
+
+def valid_tail_offset(chunk: bytes, chunk_start: int) -> int:
+    """Absolute file offset just past the last ``\\n`` in ``chunk``.
+
+    ``chunk`` is the file's final ≤64 KiB window; ``chunk_start`` is that
+    window's absolute file offset. Used by the appender to ``ftruncate`` a torn
+    tail back to the last complete line (spec §4.3). When the window holds no
+    newline at all, returns ``chunk_start`` (the whole window is one incomplete
+    line — the appender treats a >64 KiB such window as a hard error)."""
+    idx = chunk.rfind(b"\n")
+    if idx == -1:
+        return chunk_start
+    return chunk_start + idx + 1
+
+
+# --------------------------------------------------------------------------
+# decode helper
+# --------------------------------------------------------------------------
+
+def iter_decoded(lines):
+    """Yield ``(offset, decode_line(raw))`` for each ``(offset, raw)`` in
+    ``lines`` — a thin pairing helper so the ingester can count malformed lines
+    (``None`` results) while keeping their offsets for diagnostics."""
+    for offset, raw in lines:
+        yield offset, decode_line(raw)

package/bin/cctally

@@ -938,12 +938,15 @@ DowngradeDetected = _cctally_db.DowngradeDetected
 ProdMigrationRefused = _cctally_db.ProdMigrationRefused
 StatsDbCorruptError = _cctally_db.StatsDbCorruptError
 StatsDbMaintenanceError = _cctally_db.StatsDbMaintenanceError
+StatsEpochMismatchError = _cctally_db.StatsEpochMismatchError
 _is_sqlite_corruption_error = _cctally_db._is_sqlite_corruption_error
 _stats_corruption_guidance = _cctally_db._stats_corruption_guidance
 _STATS_MIGRATIONS = _cctally_db._STATS_MIGRATIONS
 _CACHE_MIGRATIONS = _cctally_db._CACHE_MIGRATIONS
+_CONVERSATIONS_MIGRATIONS = _cctally_db._CONVERSATIONS_MIGRATIONS
 _make_migration_decorator = _cctally_db._make_migration_decorator
 stats_migration = _cctally_db.stats_migration
+conversations_migration = _cctally_db.conversations_migration
 cache_migration = _cctally_db.cache_migration
 _LEGACY_MARKER_ALIASES_BY_DB = _cctally_db._LEGACY_MARKER_ALIASES_BY_DB
 _bootstrap_rename_legacy_markers = _cctally_db._bootstrap_rename_legacy_markers
@@ -964,6 +967,7 @@ _db_path_for_label = _cctally_db._db_path_for_label
 cmd_db_skip = _cctally_db.cmd_db_skip
 cmd_db_unskip = _cctally_db.cmd_db_unskip
 cmd_db_recover = _cctally_db.cmd_db_recover
+cmd_db_rebuild = _cctally_db.cmd_db_rebuild
 cmd_db_repair = _cctally_db.cmd_db_repair
 cmd_db_backup = _cctally_db.cmd_db_backup
 cmd_db_checkpoint = _cctally_db.cmd_db_checkpoint
@@ -1058,7 +1062,7 @@ _resolve_prior_5h = _cctally_record._resolve_prior_5h
 _resolve_active_five_hour_reset_event_id = _cctally_record._resolve_active_five_hour_reset_event_id
 _apply_credit = _cctally_record._apply_credit
 _fire_in_place_credit = _cctally_record._fire_in_place_credit
-_force_clear_credit = _cctally_record._force_clear_credit
+detect_reset_and_credit = _cctally_record.detect_reset_and_credit
 _count_stale_replays = _cctally_record._count_stale_replays
 _credit_preview_text = _cctally_record._credit_preview_text
 _credit_json = _cctally_record._credit_json
@@ -3257,6 +3261,12 @@ def main(argv: list[str] | None = None) -> int:
     except StatsDbMaintenanceError as exc:
         eprint(f"cctally: {exc}")
         return 3
+    except StatsEpochMismatchError as exc:
+        # A version-mismatched stats.db with no journal to rebuild from — hard
+        # error with guidance (DB journal redesign §7.1). Caught before the
+        # generic DatabaseError so it never renders as a raw traceback.
+        eprint(f"cctally: {exc}")
+        return 3
     except sqlite3.DatabaseError as exc:
         if _is_sqlite_corruption_error(exc):
             eprint(