cctally 1.28.0 → 1.29.0

package/bin/_lib_conversation_query.py

@@ -29,6 +29,25 @@ def _project_label(cwd) -> str:
     return os.path.basename(cwd.rstrip("/")) or cwd
 
 
+def _subagent_key(source_path):
+    """Privacy-safe subagent-thread identity for the reader. Each subagent (Task)
+    invocation writes its own ``agent-<hash>.jsonl``; the main session is
+    ``<session_id>.jsonl``. Returns the agent hash (``agent-`` prefix + ``.jsonl``
+    suffix stripped; an ``acompact-`` middle is kept), or ``None`` for the main
+    file / a non-agent path. We expose ONLY this derived key — never the raw
+    absolute ``source_path`` (which leaks home dir / username / encoded project,
+    and the conversation routes are LAN-exposable via dashboard.expose_transcripts)."""
+    if not source_path:
+        return None
+    base = os.path.basename(source_path)
+    if not base.startswith("agent-"):
+        return None
+    stem = base[len("agent-"):]
+    if stem.endswith(".jsonl"):
+        stem = stem[: -len(".jsonl")]
+    return stem or None
+
+
 def _entry_cost(model, inp, out, cc, cr, cost_usd_raw) -> float:
     """Cost for one session_entries row via the shared pricing helper. Tokens →
     the helper's usage dict. cost_usd_raw is passed as the optional override the
@@ -198,7 +217,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     # uuid, so the first occurrence in ascending order is canonical.
     raw = conn.execute(
         "SELECT id, uuid, timestamp_utc, entry_type, text, blocks_json, model, "
-        "       msg_id, req_id, is_sidechain, cwd, git_branch "
+        "       msg_id, req_id, is_sidechain, cwd, git_branch, source_path, parent_uuid "
         "FROM conversation_messages WHERE session_id=? "
         "ORDER BY timestamp_utc, id", (session_id,)).fetchall()
 
@@ -224,7 +243,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     turn_index = {}        # (msg_id, req_id) -> index into items
     for row in logical:
         (rid, u, ts, etype, text, blocks, model, msg_id, req_id,
-         is_sc, cwd, branch) = row
+         is_sc, cwd, branch, source_path, parent_uuid) = row
         if etype == "assistant" and msg_id is not None:
             key = (msg_id, req_id)
             idx = turn_index.get(key)
@@ -327,6 +346,12 @@ def _build_turn(members):
         "blocks": [],
         "model": first[6],
         "is_sidechain": bool(first[9]),
+        # subagent_key / parent_uuid are SEED-sourced (the first fragment, the
+        # turn's entry point) and NOT re-promoted in _fold_fragment — the prose
+        # anchor's parent_uuid is an intra-turn link, not the entry point (Codex
+        # P1). subagent_key is uniform across a turn's fragments (one file).
+        "subagent_key": _subagent_key(first[12]),
+        "parent_uuid": first[13],
         "_msg_id": first[7],
         "_req_id": first[8],
         "_has_prose": False,
@@ -374,7 +399,8 @@ def _build_simple(row):
     key → no session_entries join); it carries an explicit cost_usd of 0.0 and NO
     internal _msg_id/_req_id keys, so the cost loop's KeyError path can never fire
     (I2). The model is preserved for assistant rows."""
-    (rid, u, ts, etype, text, blocks, model, msg_id, req_id, is_sc, cwd, branch) = row
+    (rid, u, ts, etype, text, blocks, model, msg_id, req_id, is_sc, cwd, branch,
+     source_path, parent_uuid) = row
     try:
         parsed = _json.loads(blocks or "[]")
     except (ValueError, TypeError):
@@ -387,6 +413,8 @@ def _build_simple(row):
         "text": text,
         "blocks": parsed,
         "is_sidechain": bool(is_sc),
+        "subagent_key": _subagent_key(source_path),
+        "parent_uuid": parent_uuid,
     }
     if etype == "assistant":
         item["model"] = model
@@ -440,19 +468,6 @@ def _row_to_hit(uuid_, sid, ts, cwd, snippet, msg_id, req_id):
     }
 
 
-def _dedup_hits(hits, limit, offset):
-    seen = set()
-    out = []
-    for h in hits:
-        key = (h["session_id"], h["uuid"])
-        if key in seen:
-            continue
-        seen.add(key)
-        out.append(h)
-    total = len(out)
-    return out[offset:offset + limit], total
-
-
 def _attach_costs(conn, page):
     """Compute turn cost for the FINAL page's hits in ONE _turn_cost_map call,
     then map it onto each hit and drop the private `_turn_key`. Off-page and
@@ -465,45 +480,126 @@ def _attach_costs(conn, page):
     return page
 
 
-def _search_fts(conn, q, limit, offset):
-    sql = (
-        "SELECT cm.session_id, cm.uuid, cm.timestamp_utc, cm.cwd, "
-        "       cm.msg_id, cm.req_id, "
-        "       snippet(conversation_fts, 0, '[', ']', ' … ', 12) AS snip "
+def _like_pattern(q):
+    """Build the LIKE pattern for `q`. Escape the ESCAPE char (\\) FIRST, then
+    the wildcards — otherwise a query containing a backslash (incl. a trailing
+    one) mis-escapes the appended '%' and the LIKE silently matches nothing
+    (paired with ESCAPE '\\' in the queries below)."""
+    return ("%" + q.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
+            + "%")
+
+
+def _fts_snippets(conn, fts_q, ids):
+    """{rowid: snippet} for the page rowids ONLY (#149). snippet() needs an
+    active MATCH, so it can't be deferred to an outer query over the page CTE;
+    a second bounded MATCH restricted to the page rowids generates snippets for
+    at most one page of hits instead of every corpus match."""
+    if not ids:
+        return {}
+    ph = ",".join("?" for _ in ids)
+    rows = conn.execute(
+        "SELECT cm.id, snippet(conversation_fts, 0, '[', ']', ' … ', 12) "
         "FROM conversation_fts "
         "JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
-        "WHERE conversation_fts MATCH ? "
-        # cm.id is the final tiebreaker so equal (rank, timestamp) hits order
-        # deterministically — _dedup_hits keeps the FIRST occurrence, so without
-        # it the surviving snippet/cost (and page boundary) would flip run-to-run.
-        "ORDER BY bm25(conversation_fts), cm.timestamp_utc DESC, cm.id DESC"
-    )
-    raw = conn.execute(sql, (_fts_query(q),)).fetchall()
-    hits = [_row_to_hit(u, sid, ts, cwd, snip, mid, rqd)
-            for (sid, u, ts, cwd, mid, rqd, snip) in raw]
-    page, total = _dedup_hits(hits, limit, offset)
-    return {"query": q, "mode": "fts", "hits": _attach_costs(conn, page),
+        f"WHERE conversation_fts MATCH ? AND cm.id IN ({ph})",
+        (fts_q, *ids),
+    ).fetchall()
+    return {r[0]: r[1] for r in rows}
+
+
+def _texts_for_ids(conn, ids):
+    """{rowid: text} for the page rowids ONLY (#149) — the LIKE page query omits
+    `text` so we never pull every matched row's body into Python; this fetches
+    it for just the page so `_manual_snippet` runs at most `limit` times."""
+    if not ids:
+        return {}
+    ph = ",".join("?" for _ in ids)
+    rows = conn.execute(
+        f"SELECT id, text FROM conversation_messages WHERE id IN ({ph})",
+        tuple(ids),
+    ).fetchall()
+    return {r[0]: r[1] for r in rows}
+
+
+def _search_fts(conn, q, limit, offset):
+    # All of dedup + paging + total live in SQL (#149) so Python never holds
+    # more than one page of hits/snippets, regardless of corpus match count.
+    fts_q = _fts_query(q)
+    # Exact post-dedup logical total — counted in C with no snippet generation
+    # and no Python row materialization.
+    total = conn.execute(
+        "SELECT COUNT(*) FROM ("
+        "  SELECT DISTINCT cm.session_id, cm.uuid "
+        "  FROM conversation_fts "
+        "  JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
+        "  WHERE conversation_fts MATCH ?)",
+        (fts_q,),
+    ).fetchone()[0]
+    # One row per logical (session_id, uuid): ROW_NUMBER()=1 keeps the SAME row
+    # the old Python dedup kept as its FIRST occurrence (order: bm25, ts DESC,
+    # id DESC — cm.id is the final deterministic tiebreaker), so the surviving
+    # snippet/cost and the page boundary stay byte-stable. bm25 still ranks
+    # across all matches (inherent to relevance ordering).
+    #
+    # bm25 is materialized as a plain `rank` column in the inner `matched` CTE
+    # before the window function runs: FTS5 auxiliary functions (bm25/snippet)
+    # may only be used directly against the MATCH query, NOT inside a window
+    # ORDER BY ("unable to use function bm25 in the requested context").
+    page = conn.execute(
+        "WITH matched AS ("
+        "  SELECT cm.id AS rid, cm.session_id AS sid, cm.uuid AS uuid, "
+        "         cm.timestamp_utc AS ts, cm.cwd AS cwd, "
+        "         cm.msg_id AS mid, cm.req_id AS rqd, "
+        "         bm25(conversation_fts) AS rank "
+        "  FROM conversation_fts "
+        "  JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
+        "  WHERE conversation_fts MATCH ?), "
+        "ranked AS ("
+        "  SELECT *, ROW_NUMBER() OVER ("
+        "             PARTITION BY sid, uuid ORDER BY rank, ts DESC, rid DESC"
+        "           ) AS rn "
+        "  FROM matched) "
+        "SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
+        "ORDER BY rank, ts DESC, rid DESC LIMIT ? OFFSET ?",
+        (fts_q, limit, offset),
+    ).fetchall()
+    snips = _fts_snippets(conn, fts_q, [r[0] for r in page])
+    hits = [_row_to_hit(uuid, sid, ts, cwd, snips.get(rid, ""), mid, rqd)
+            for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
+    return {"query": q, "mode": "fts", "hits": _attach_costs(conn, hits),
             "total": total}
 
 
 def _search_like(conn, q, limit, offset):
-    # Escape the ESCAPE char (\) FIRST, then the wildcards — otherwise a query
-    # containing a backslash (incl. a trailing one) mis-escapes the appended
-    # '%' and the LIKE silently matches nothing (ESCAPE '\' below).
-    like = ("%" + q.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
-            + "%")
-    sql = (
-        "SELECT session_id, uuid, timestamp_utc, cwd, msg_id, req_id, text "
-        "FROM conversation_messages "
-        "WHERE text LIKE ? ESCAPE '\\' AND text != '' "
-        "ORDER BY timestamp_utc DESC, id DESC"
-    )
-    hits = []
-    for sid, u, ts, cwd, mid, rqd, text in conn.execute(sql, (like,)):
-        hits.append(_row_to_hit(u, sid, ts, cwd,
-                                _manual_snippet(text, q), mid, rqd))
-    page, total = _dedup_hits(hits, limit, offset)
-    return {"query": q, "mode": "like", "hits": _attach_costs(conn, page),
+    # SQL-bounded mirror of _search_fts for the no-FTS5 fallback (#149); the
+    # COUNT + page each scan the table once (the degraded path already lacks an
+    # index for the substring match).
+    like = _like_pattern(q)
+    total = conn.execute(
+        "SELECT COUNT(*) FROM ("
+        "  SELECT DISTINCT session_id, uuid FROM conversation_messages "
+        "  WHERE text LIKE ? ESCAPE '\\' AND text != '')",
+        (like,),
+    ).fetchone()[0]
+    page = conn.execute(
+        "WITH ranked AS ("
+        "  SELECT id AS rid, session_id AS sid, uuid AS uuid, "
+        "         timestamp_utc AS ts, cwd AS cwd, msg_id AS mid, req_id AS rqd, "
+        "         ROW_NUMBER() OVER ("
+        "           PARTITION BY session_id, uuid "
+        "           ORDER BY timestamp_utc DESC, id DESC"
+        "         ) AS rn "
+        "  FROM conversation_messages "
+        "  WHERE text LIKE ? ESCAPE '\\' AND text != '') "
+        "SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
+        "ORDER BY ts DESC, rid DESC LIMIT ? OFFSET ?",
+        (like, limit, offset),
+    ).fetchall()
+    texts = _texts_for_ids(conn, [r[0] for r in page])
+    hits = [_row_to_hit(uuid, sid, ts, cwd,
+                        _manual_snippet(texts.get(rid, ""), q), mid, rqd)
+            for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
+    return {"query": q, "mode": "like", "hits": _attach_costs(conn, hits),
             "total": total}
 
 

package/bin/_lib_jsonl.py

@@ -201,6 +201,68 @@ def _parse_usage_entries(
     return no_key_entries
 
 
+def parse_cost_entry(obj, path_str: str):
+    """Pure per-line cost parser: given a parsed JSONL object, return
+    ``(UsageEntry, msg_id, req_id)`` when it is a billable assistant entry, or
+    ``None`` otherwise (non-assistant, missing/invalid usage, model, or
+    timestamp, or a ``<synthetic>`` placeholder). No I/O, no byte offset — the
+    caller owns the readline()+tell() loop.
+
+    Extracted (#138) so the streaming ``_iter_jsonl_entries_with_offsets`` reader
+    and the fused single-pass sync walker (``_cctally_cache._iter_sync_entries``)
+    share ONE gating implementation — each JSONL line is ``json.loads``-parsed
+    once and classified once, never re-parsed for a separate second walk.
+    """
+    if obj.get("type") != "assistant":
+        return None
+
+    ts_raw = obj.get("timestamp")
+    if not isinstance(ts_raw, str) or not ts_raw.strip():
+        return None
+
+    msg = obj.get("message")
+    if not isinstance(msg, dict):
+        msg = obj
+
+    usage = msg.get("usage")
+    if not isinstance(usage, dict):
+        return None
+
+    model = msg.get("model") or obj.get("model")
+    if not isinstance(model, str) or not model.strip():
+        return None
+    model = model.strip()
+    if model == "<synthetic>":
+        # Matches ccusage's claude_loader.rs:454. Filtered here so the cache
+        # ingest path can't accidentally store these rows even if a downstream
+        # loop forgets to double-check (see `sync_cache` in _cctally_cache.py).
+        return None
+
+    try:
+        ts = dt.datetime.fromisoformat(ts_raw.strip().replace("Z", "+00:00"))
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=dt.timezone.utc)
+    except ValueError:
+        return None
+
+    msg_id = msg.get("id")
+    req_id = obj.get("requestId")
+    cost_usd_raw = obj.get("costUSD")
+    cost_usd = float(cost_usd_raw) if cost_usd_raw is not None else None
+
+    return (
+        UsageEntry(
+            timestamp=ts,
+            model=model,
+            usage=usage,
+            cost_usd=cost_usd,
+            source_path=path_str,
+        ),
+        msg_id,
+        req_id,
+    )
+
+
 def _iter_jsonl_entries_with_offsets(fh, path_str: str):
     """Yield (byte_offset, UsageEntry, msg_id, req_id) for each assistant
     entry starting from fh's current position.
@@ -209,7 +271,9 @@ def _iter_jsonl_entries_with_offsets(fh, path_str: str):
     accurate for resume-from-offset after partial ingests. Malformed JSON
     and non-assistant lines are skipped, but the offset still advances past
     them so they are never re-read. Range filtering is intentionally NOT
-    done here — filters are applied at query time by iter_entries().
+    done here — filters are applied at query time by iter_entries(). The
+    per-line gating lives in ``parse_cost_entry`` (shared with the fused
+    single-pass sync walker, #138).
     """
     while True:
         offset = fh.tell()
@@ -230,56 +294,11 @@ def _iter_jsonl_entries_with_offsets(fh, path_str: str):
             obj = json.loads(stripped)
         except json.JSONDecodeError:
             continue
-        if obj.get("type") != "assistant":
-            continue
-
-        ts_raw = obj.get("timestamp")
-        if not isinstance(ts_raw, str) or not ts_raw.strip():
-            continue
-
-        msg = obj.get("message")
-        if not isinstance(msg, dict):
-            msg = obj
-
-        usage = msg.get("usage")
-        if not isinstance(usage, dict):
-            continue
-
-        model = msg.get("model") or obj.get("model")
-        if not isinstance(model, str) or not model.strip():
+        parsed = parse_cost_entry(obj, path_str)
+        if parsed is None:
             continue
-        model = model.strip()
-        if model == "<synthetic>":
-            # Matches ccusage's claude_loader.rs:454. Filtered at the
-            # iterator level so the cache ingest path can't accidentally
-            # store these rows even if a downstream loop forgets to
-            # double-check (see `sync_cache` in _cctally_cache.py).
-            continue
-
-        try:
-            ts = dt.datetime.fromisoformat(ts_raw.strip().replace("Z", "+00:00"))
-            if ts.tzinfo is None:
-                ts = ts.replace(tzinfo=dt.timezone.utc)
-        except ValueError:
-            continue
-
-        msg_id = msg.get("id")
-        req_id = obj.get("requestId")
-        cost_usd_raw = obj.get("costUSD")
-        cost_usd = float(cost_usd_raw) if cost_usd_raw is not None else None
-
-        yield (
-            offset,
-            UsageEntry(
-                timestamp=ts,
-                model=model,
-                usage=usage,
-                cost_usd=cost_usd,
-                source_path=path_str,
-            ),
-            msg_id,
-            req_id,
-        )
+        entry, msg_id, req_id = parsed
+        yield (offset, entry, msg_id, req_id)
 
 
 _CODEX_FILENAME_UUID_RE = re.compile(

package/bin/cctally

@@ -2100,18 +2100,18 @@ get_max_milestone_for_week          = _cctally_milestones.get_max_milestone_for_
 get_milestone_cost_for_week         = _cctally_milestones.get_milestone_cost_for_week         # record shim
 get_milestones_for_week             = _cctally_milestones.get_milestones_for_week             # forecast c.; tui shim; percent-breakdown c.
 insert_percent_milestone            = _cctally_milestones.insert_percent_milestone            # record shim; idempotency-test mod.
-insert_budget_milestone             = _cctally_milestones.insert_budget_milestone             # record shim
+insert_budget_milestone             = _cctally_milestones.insert_budget_milestone             # record shim; test_budget_alerts / test_project_budget_dashboard ns[] (+ test_codex_budget_alerts / test_projected_alerts post-#143 vendor-param unification)
 insert_project_budget_milestone     = _cctally_milestones.insert_project_budget_milestone     # record shim; project-budget-config-test ns[]
-insert_codex_budget_milestone       = _cctally_milestones.insert_codex_budget_milestone       # record shim; test_codex_budget_alerts ns[]
-_codex_budget_crossings             = _cctally_milestones._codex_budget_crossings             # record shim (shared INSERT-and-arm core for the codex_budget axis)
+_budget_crossings                   = _cctally_milestones._budget_crossings                   # record shim (shared INSERT-and-arm core for the budget axis, both vendors, #143)
 _resolve_codex_budget_period_window = _cctally_milestones._resolve_codex_budget_period_window # record shim; milestones c. (codex period window)
-_reconcile_codex_budget_milestones_on_set = _cctally_milestones._reconcile_codex_budget_milestones_on_set  # test_codex_budget_alerts ns[]; forecast set/reconcile
+_resolve_budget_window              = _cctally_milestones._resolve_budget_window              # record shim; milestones c. (per-vendor cheap budget window dispatcher, #143)
+_budget_spend_for_vendor            = _cctally_milestones._budget_spend_for_vendor            # record shim; milestones c. (per-vendor budget spend dispatcher, #143)
 _reconcile_codex_budget_on_config_write = _cctally_milestones._reconcile_codex_budget_on_config_write  # forecast/config c. (forward-only codex-budget reconcile)
 _resolve_claude_budget_window       = _cctally_milestones._resolve_claude_budget_window       # record shim; milestones c. (period-aware Claude budget window)
 _project_crossings                  = _cctally_milestones._project_crossings                  # record shim; milestones c. (#130 firing/reconcile shared crossing arithmetic)
 insert_projected_milestone          = _cctally_milestones.insert_projected_milestone          # record shim
 _projected_levels_already_latched   = _cctally_milestones._projected_levels_already_latched   # record shim
-_reconcile_budget_milestones_on_set = _cctally_milestones._reconcile_budget_milestones_on_set # test_budget_alerts ns[]
+_reconcile_budget_milestones_on_set = _cctally_milestones._reconcile_budget_milestones_on_set # test_budget_alerts / test_codex_budget_alerts ns[] (vendor-param, #143)
 _reconcile_budget_on_config_write   = _cctally_milestones._reconcile_budget_on_config_write   # forecast/config/dashboard c.; test_forecast_ns_patch mod. patch
 _reconcile_project_budget_milestones_on_write = _cctally_milestones._reconcile_project_budget_milestones_on_write  # forecast/config/dashboard c. (forward-only project-budget reconcile)