cctally 1.80.4 → 1.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/_cctally_account.py +397 -0
- package/bin/_cctally_alerts.py +58 -2
- package/bin/_cctally_cache.py +696 -166
- package/bin/_cctally_cache_report.py +21 -3
- package/bin/_cctally_config.py +119 -8
- package/bin/_cctally_core.py +343 -48
- package/bin/_cctally_dashboard.py +32 -7
- package/bin/_cctally_dashboard_conversation.py +6 -2
- package/bin/_cctally_dashboard_envelope.py +49 -0
- package/bin/_cctally_dashboard_share.py +78 -1
- package/bin/_cctally_dashboard_sources.py +434 -48
- package/bin/_cctally_db.py +659 -152
- package/bin/_cctally_diff.py +9 -0
- package/bin/_cctally_doctor.py +201 -26
- package/bin/_cctally_five_hour.py +110 -69
- package/bin/_cctally_forecast.py +112 -43
- package/bin/_cctally_journal.py +591 -80
- package/bin/_cctally_milestones.py +180 -67
- package/bin/_cctally_parser.py +87 -0
- package/bin/_cctally_percent_breakdown.py +24 -15
- package/bin/_cctally_project.py +12 -1
- package/bin/_cctally_quota.py +150 -39
- package/bin/_cctally_record.py +491 -141
- package/bin/_cctally_reporting.py +66 -14
- package/bin/_cctally_setup.py +9 -1
- package/bin/_cctally_source_analytics.py +56 -7
- package/bin/_cctally_statusline.py +55 -8
- package/bin/_cctally_store.py +20 -8
- package/bin/_cctally_sync_week.py +30 -10
- package/bin/_cctally_tui.py +99 -18
- package/bin/_cctally_weekrefs.py +38 -15
- package/bin/_lib_accounts.py +325 -0
- package/bin/_lib_alerts_payload.py +25 -4
- package/bin/_lib_cache_writer_lock.py +77 -0
- package/bin/_lib_codex_hooks.py +40 -1
- package/bin/_lib_diff_kernel.py +28 -14
- package/bin/_lib_doctor.py +160 -0
- package/bin/_lib_journal.py +65 -2
- package/bin/_lib_quota.py +81 -4
- package/bin/_lib_render.py +19 -5
- package/bin/_lib_share.py +25 -0
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_subscription_weeks.py +16 -2
- package/bin/_lib_view_models.py +18 -9
- package/bin/cctally +43 -4
- package/dashboard/static/assets/index-DJP4gEB7.js +92 -0
- package/dashboard/static/assets/index-Dk1nplOz.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-BzWujBS3.js +0 -92
- package/dashboard/static/assets/index-Cc2ykqF_.css +0 -1
package/bin/_cctally_cache.py
CHANGED
|
@@ -103,6 +103,7 @@ import fcntl
|
|
|
103
103
|
import json
|
|
104
104
|
import os
|
|
105
105
|
import pathlib
|
|
106
|
+
import signal
|
|
106
107
|
import sqlite3
|
|
107
108
|
import sys
|
|
108
109
|
import time
|
|
@@ -371,6 +372,21 @@ CHECKPOINT_AUTO_BUSY_TIMEOUT_MS = 100
|
|
|
371
372
|
CHECKPOINT_CMD_BUSY_TIMEOUT_MS = 15000
|
|
372
373
|
|
|
373
374
|
|
|
375
|
+
def _cache_storm_test_pause(point: str) -> None:
|
|
376
|
+
"""Private process-control seam for the killed-writer stress harness.
|
|
377
|
+
|
|
378
|
+
Production is a zero-cost string comparison. Tests arm one exact point and
|
|
379
|
+
marker path, then SIGKILL the stopped child from the parent process.
|
|
380
|
+
"""
|
|
381
|
+
if os.environ.get("CCTALLY_TEST_CACHE_STORM_PAUSE_AT") != point:
|
|
382
|
+
return
|
|
383
|
+
marker = os.environ.get("CCTALLY_TEST_CACHE_STORM_MARKER")
|
|
384
|
+
if not marker:
|
|
385
|
+
return
|
|
386
|
+
pathlib.Path(marker).write_text(f"{os.getpid()}\n")
|
|
387
|
+
os.kill(os.getpid(), signal.SIGSTOP)
|
|
388
|
+
|
|
389
|
+
|
|
374
390
|
class CheckpointResult(NamedTuple):
|
|
375
391
|
"""Outcome of a single ``PRAGMA wal_checkpoint(TRUNCATE)`` (#297).
|
|
376
392
|
|
|
@@ -430,8 +446,18 @@ def _maybe_truncate_wal(conn, db_path) -> None:
|
|
|
430
446
|
all ingest work by this point.
|
|
431
447
|
"""
|
|
432
448
|
try:
|
|
433
|
-
|
|
449
|
+
trigger = CACHE_WAL_CHECKPOINT_TRIGGER_BYTES
|
|
450
|
+
test_trigger = os.environ.get("CCTALLY_TEST_CACHE_WAL_TRIGGER_BYTES")
|
|
451
|
+
if test_trigger is not None:
|
|
452
|
+
try:
|
|
453
|
+
trigger = max(0, int(test_trigger))
|
|
454
|
+
except ValueError:
|
|
455
|
+
pass
|
|
456
|
+
if _wal_file_size(db_path) <= trigger:
|
|
434
457
|
return
|
|
458
|
+
# After this point, resuming the private stopped process necessarily
|
|
459
|
+
# enters a real TRUNCATE checkpoint rather than a threshold no-op.
|
|
460
|
+
_cache_storm_test_pause("cache_precheckpoint")
|
|
435
461
|
prior = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
|
436
462
|
try:
|
|
437
463
|
conn.execute(f"PRAGMA busy_timeout={CHECKPOINT_AUTO_BUSY_TIMEOUT_MS}")
|
|
@@ -688,6 +714,101 @@ def _codex_provider_roots() -> list[CodexProviderRoot]:
|
|
|
688
714
|
return roots
|
|
689
715
|
|
|
690
716
|
|
|
717
|
+
@dataclass(frozen=True)
|
|
718
|
+
class _CodexRootAccount:
|
|
719
|
+
"""Resolved account for one Codex provider root (spec §1 observe-and-stamp).
|
|
720
|
+
|
|
721
|
+
``status`` ∈ {"identified", "stably_absent", "torn"}; ``account_key`` is the
|
|
722
|
+
real key ONLY when identified, with ``identity`` carrying the registry
|
|
723
|
+
enrichment (natural_id/email/plan_type). stably-absent (missing auth.json /
|
|
724
|
+
api-key mode) stamps NULL (``NULL ≡ unattributed`` on the read path); torn
|
|
725
|
+
defers the whole root's files this cycle without advancing any cursor.
|
|
726
|
+
"""
|
|
727
|
+
|
|
728
|
+
status: str
|
|
729
|
+
account_key: str | None = None
|
|
730
|
+
identity: dict | None = None
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def _read_codex_account_from_auth_bytes(data: bytes) -> "dict | None":
|
|
734
|
+
"""stable-read reader for ``auth.json``: bytes -> identity dict or None.
|
|
735
|
+
|
|
736
|
+
Raises :class:`_lib_accounts.TornRead` on unparseable/half-written JSON so
|
|
737
|
+
the stable-read machinery reports *torn* (defer). Returns None for api-key
|
|
738
|
+
mode / no ChatGPT identity (-> stably_absent). The id_token lives at
|
|
739
|
+
``tokens.id_token`` (official Codex CLI shape); a flat ``id_token`` is
|
|
740
|
+
tolerated. NO signature verification — we read our own disk state. The dict
|
|
741
|
+
carries ``{account_key, natural_id, email, plan_type}`` for the registry."""
|
|
742
|
+
import _lib_accounts
|
|
743
|
+
try:
|
|
744
|
+
obj = json.loads(data)
|
|
745
|
+
except (ValueError, TypeError):
|
|
746
|
+
raise _lib_accounts.TornRead()
|
|
747
|
+
if not isinstance(obj, dict):
|
|
748
|
+
raise _lib_accounts.TornRead()
|
|
749
|
+
tokens = obj.get("tokens")
|
|
750
|
+
id_token = tokens.get("id_token") if isinstance(tokens, dict) else obj.get("id_token")
|
|
751
|
+
payload = _lib_accounts.decode_id_token_payload(id_token)
|
|
752
|
+
natural = _lib_accounts.codex_natural_id(payload)
|
|
753
|
+
if natural is None:
|
|
754
|
+
return None
|
|
755
|
+
return {
|
|
756
|
+
"account_key": _lib_accounts.account_key("codex", natural),
|
|
757
|
+
"natural_id": natural,
|
|
758
|
+
"email": _lib_accounts.codex_email(payload),
|
|
759
|
+
"plan_type": _lib_accounts.codex_plan_type(payload),
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def _resolve_codex_account_for_root(provider_root: pathlib.Path) -> _CodexRootAccount:
|
|
764
|
+
"""Resolve one provider root's active account via a stable-read of its own
|
|
765
|
+
``<provider_root>/auth.json`` (spec §1 — attribution is per-root, so a future
|
|
766
|
+
per-account-roots layout gets deterministic attribution for free)."""
|
|
767
|
+
import _lib_accounts
|
|
768
|
+
path = str(provider_root / "auth.json")
|
|
769
|
+
result = _lib_accounts.stable_read_identity(
|
|
770
|
+
path, _read_codex_account_from_auth_bytes)
|
|
771
|
+
if result.status == "identified":
|
|
772
|
+
info = result.value
|
|
773
|
+
return _CodexRootAccount("identified", str(info["account_key"]), info)
|
|
774
|
+
if result.status == "torn":
|
|
775
|
+
return _CodexRootAccount("torn", None)
|
|
776
|
+
return _CodexRootAccount("stably_absent", None)
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
_OBSERVED_CODEX_ACCOUNT_MARKER_PREFIX = "observed-codex-account-"
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def _maybe_append_codex_account_observe(identity: "dict | None") -> None:
|
|
783
|
+
"""Append an ``account_observe`` op on first sight of a Codex account or an
|
|
784
|
+
identity change (#341, spec §1) — NOT every sync. Deduped by a per-account
|
|
785
|
+
marker file in APP_DIR (per-account, since one machine hosts multiple Codex
|
|
786
|
+
accounts — a scalar marker would flip-flop). Best-effort: a marker/journal
|
|
787
|
+
hiccup never breaks the ingest."""
|
|
788
|
+
import _lib_accounts
|
|
789
|
+
if not identity:
|
|
790
|
+
return
|
|
791
|
+
account_key = identity.get("account_key")
|
|
792
|
+
if not account_key or account_key == _lib_accounts.UNATTRIBUTED:
|
|
793
|
+
return
|
|
794
|
+
marker = (_cctally_core.APP_DIR
|
|
795
|
+
/ (_OBSERVED_CODEX_ACCOUNT_MARKER_PREFIX + account_key))
|
|
796
|
+
if marker.exists():
|
|
797
|
+
return
|
|
798
|
+
try:
|
|
799
|
+
import _cctally_journal as _jr
|
|
800
|
+
import _lib_journal as _lj
|
|
801
|
+
at = (_cctally_core._command_as_of()
|
|
802
|
+
.isoformat(timespec="seconds").replace("+00:00", "Z"))
|
|
803
|
+
_jr.append_record(_lj.make_account_observe(
|
|
804
|
+
at=at, account_key=account_key, provider="codex",
|
|
805
|
+
natural_id=identity.get("natural_id"), email=identity.get("email"),
|
|
806
|
+
plan_type=identity.get("plan_type"), label_source="auto"))
|
|
807
|
+
marker.write_text(account_key + "\n")
|
|
808
|
+
except OSError:
|
|
809
|
+
pass
|
|
810
|
+
|
|
811
|
+
|
|
691
812
|
def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
|
|
692
813
|
"""Discover each physical rollout once with the first matching root facts."""
|
|
693
814
|
discovered: list[CodexDiscoveredFile] = []
|
|
@@ -1224,15 +1345,21 @@ def _append_codex_quota_obs(quota_rows: list) -> None:
|
|
|
1224
1345
|
(source, source_root_key, source_path, line_offset, captured_at_utc,
|
|
1225
1346
|
observed_slot, logical_limit_key, limit_id, limit_name, window_minutes,
|
|
1226
1347
|
used_percent, resets_at_utc, plan_type, individual_limit_json,
|
|
1227
|
-
reached_type, observed_model) = row
|
|
1348
|
+
reached_type, observed_model, account_key) = row
|
|
1228
1349
|
at = captured_at_utc or (
|
|
1229
1350
|
_cctally_core._command_as_of()
|
|
1230
1351
|
.isoformat(timespec="seconds")
|
|
1231
1352
|
.replace("+00:00", "Z")
|
|
1232
1353
|
)
|
|
1233
1354
|
try:
|
|
1355
|
+
# #341: the obs carries the top-level ``account`` field ONLY for a
|
|
1356
|
+
# real account (invariant: the sentinel/single-account case OMITS the
|
|
1357
|
+
# field entirely, so a no-auth install produces byte-identical
|
|
1358
|
+
# journals). NULL/unattributed re-derives to NULL account_key at the
|
|
1359
|
+
# QUOTA_APPLIER's cache upsert.
|
|
1234
1360
|
_jr.append_record(_jl.make_obs(
|
|
1235
1361
|
at=at, src="codex-quota", provider="codex",
|
|
1362
|
+
account=account_key,
|
|
1236
1363
|
payload={
|
|
1237
1364
|
"kind": "quota_window_snapshot",
|
|
1238
1365
|
"source": source, "source_root_key": source_root_key,
|
|
@@ -1272,6 +1399,7 @@ def _write_codex_file_batch(
|
|
|
1272
1399
|
thread_rows: list[tuple[Any, ...]],
|
|
1273
1400
|
active_root_keys: set[str],
|
|
1274
1401
|
prune_roots: bool = True,
|
|
1402
|
+
account_key: "str | None" = None,
|
|
1275
1403
|
) -> int:
|
|
1276
1404
|
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
1277
1405
|
|
|
@@ -1299,8 +1427,8 @@ def _write_codex_file_batch(
|
|
|
1299
1427
|
(source_path, line_offset, timestamp_utc, session_id, model,
|
|
1300
1428
|
input_tokens, cached_input_tokens, output_tokens,
|
|
1301
1429
|
reasoning_output_tokens, total_tokens, source_root_key,
|
|
1302
|
-
conversation_key)
|
|
1303
|
-
VALUES (
|
|
1430
|
+
conversation_key, account_key)
|
|
1431
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
1304
1432
|
accounting_rows,
|
|
1305
1433
|
)
|
|
1306
1434
|
rows_changed = conn.total_changes - before
|
|
@@ -1310,8 +1438,9 @@ def _write_codex_file_batch(
|
|
|
1310
1438
|
(source, source_root_key, source_path, line_offset,
|
|
1311
1439
|
captured_at_utc, observed_slot, logical_limit_key, limit_id,
|
|
1312
1440
|
limit_name, window_minutes, used_percent, resets_at_utc,
|
|
1313
|
-
plan_type, individual_limit_json, reached_type, observed_model
|
|
1314
|
-
|
|
1441
|
+
plan_type, individual_limit_json, reached_type, observed_model,
|
|
1442
|
+
account_key)
|
|
1443
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
1315
1444
|
quota_rows,
|
|
1316
1445
|
)
|
|
1317
1446
|
if thread_rows:
|
|
@@ -1340,13 +1469,13 @@ def _write_codex_file_batch(
|
|
|
1340
1469
|
(path, size_bytes, mtime_ns, last_byte_offset, last_ingested_at,
|
|
1341
1470
|
last_session_id, last_model, last_total_tokens, source_root_key,
|
|
1342
1471
|
last_native_thread_id, last_root_thread_id, last_parent_thread_id,
|
|
1343
|
-
last_conversation_key, last_turn_id)
|
|
1344
|
-
VALUES (
|
|
1472
|
+
last_conversation_key, last_turn_id, account_key)
|
|
1473
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
1345
1474
|
(
|
|
1346
1475
|
path_str, size, mtime_ns, final_offset, now_iso, last_session_id,
|
|
1347
1476
|
last_model, last_total_tokens, discovered.source_root_key,
|
|
1348
1477
|
last_native_thread_id, last_root_thread_id, last_parent_thread_id,
|
|
1349
|
-
last_conversation_key, last_turn_id,
|
|
1478
|
+
last_conversation_key, last_turn_id, account_key,
|
|
1350
1479
|
),
|
|
1351
1480
|
)
|
|
1352
1481
|
if prune_roots:
|
|
@@ -1355,6 +1484,7 @@ def _write_codex_file_batch(
|
|
|
1355
1484
|
# facts as one physical unit. Keep the version bump in that same commit so
|
|
1356
1485
|
# a rolled-back batch never appears newer to the dashboard signature.
|
|
1357
1486
|
_bump_codex_physical_mutation_seq(conn)
|
|
1487
|
+
_cache_storm_test_pause("codex_precommit")
|
|
1358
1488
|
conn.commit()
|
|
1359
1489
|
return rows_changed
|
|
1360
1490
|
|
|
@@ -1442,6 +1572,11 @@ class IngestStats:
|
|
|
1442
1572
|
# and are otherwise unaffected.
|
|
1443
1573
|
files_failed: int = 0
|
|
1444
1574
|
deferred_reason: "str | None" = None
|
|
1575
|
+
# #341: a torn ``~/.claude.json`` read (mid-rewrite) DEFERS the whole Claude
|
|
1576
|
+
# tail-ingest this sync (identity resolved once per sync); the deferred sync
|
|
1577
|
+
# advances no per-file cursor so the next sync re-reads + re-stamps rather
|
|
1578
|
+
# than guessing an account. Non-zero ⇒ the walk was skipped this cycle.
|
|
1579
|
+
files_deferred_torn: int = 0
|
|
1445
1580
|
# #279 S2 F1 parse-health counters — passive observers over the new-byte
|
|
1446
1581
|
# span this sync walked. lines_seen counts non-blank lines (malformed
|
|
1447
1582
|
# included); assistant_lines_skipped counts assistant-typed lines
|
|
@@ -1890,6 +2025,29 @@ def sync_cache(
|
|
|
1890
2025
|
stats.deferred_reason = "pending_global_flags"
|
|
1891
2026
|
return stats
|
|
1892
2027
|
|
|
2028
|
+
# #341 observe-and-stamp (spec §1): resolve the active Claude identity
|
|
2029
|
+
# ONCE per sync from ~/.claude.json (stable-read, mtime-cached). The
|
|
2030
|
+
# active account stamps every newly-ingested session_entries row (and the
|
|
2031
|
+
# session_files last-observed diagnostic). A single global identity file
|
|
2032
|
+
# governs the whole Claude tail, so a TORN read (mid-rewrite) DEFERS the
|
|
2033
|
+
# ENTIRE ingest this cycle — resolved here BEFORE the rebuild wipe / the
|
|
2034
|
+
# file walk so nothing mutates and no per-file cursor advances; the next
|
|
2035
|
+
# sync re-reads the same bytes and re-stamps rather than guessing (never
|
|
2036
|
+
# mis-stamp). identified → the real key; stably-absent (no ~/.claude.json
|
|
2037
|
+
# / api-key mode) → None (stamped NULL == unattributed on the read path,
|
|
2038
|
+
# byte-stable for the no-identity corpus). The Claude account_observe is
|
|
2039
|
+
# journaled by record-usage, not here, so sync_cache owns the cache-row
|
|
2040
|
+
# stamp only — no journal double-stamp.
|
|
2041
|
+
import _lib_accounts
|
|
2042
|
+
_claude_identity = _cctally_core._resolve_active_claude_identity()
|
|
2043
|
+
if _claude_identity.get("status") == "torn":
|
|
2044
|
+
stats.files_deferred_torn += 1
|
|
2045
|
+
stats.deferred_reason = "identity_torn"
|
|
2046
|
+
return stats
|
|
2047
|
+
_account_key = _claude_identity["account_key"]
|
|
2048
|
+
file_account_key = (
|
|
2049
|
+
None if _account_key == _lib_accounts.UNATTRIBUTED else _account_key)
|
|
2050
|
+
|
|
1893
2051
|
# Walk-complete sentinel gating (cctally-dev#93, D5b/D6b). Capture
|
|
1894
2052
|
# whether cache 001 was already applied at the moment this sync
|
|
1895
2053
|
# acquired the lock. The end-of-loop marker write is gated on this so
|
|
@@ -2579,7 +2737,15 @@ def sync_cache(
|
|
|
2579
2737
|
# file's `sync_seq` and mutation_min_ts = its own
|
|
2580
2738
|
# timestamp_utc (== the event time on insert).
|
|
2581
2739
|
sync_seq = _bump_mutation_seq(conn)
|
|
2582
|
-
|
|
2740
|
+
# #341: append the active account (resolved once per sync) as
|
|
2741
|
+
# the trailing column. First-stamp-wins — account_key is
|
|
2742
|
+
# DELIBERATELY OMITTED from the ON CONFLICT DO UPDATE SET below
|
|
2743
|
+
# (spec §2 cache.db: a resumed session replaying identical
|
|
2744
|
+
# bytes under a different account is the SAME message and keeps
|
|
2745
|
+
# the first observed stamp). ``file_account_key`` is None when
|
|
2746
|
+
# stably-absent (stamped NULL == unattributed on read).
|
|
2747
|
+
stamped_rows = [
|
|
2748
|
+
r + (sync_seq, r[2], file_account_key) for r in rows]
|
|
2583
2749
|
before = conn.total_changes
|
|
2584
2750
|
# ccusage-parity ON CONFLICT DO UPDATE: higher-token total
|
|
2585
2751
|
# wins on conflict; speed-set breaks ties. The partial
|
|
@@ -2610,8 +2776,8 @@ def sync_cache(
|
|
|
2610
2776
|
msg_id, req_id, input_tokens, output_tokens,
|
|
2611
2777
|
cache_create_tokens, cache_read_tokens,
|
|
2612
2778
|
usage_extra_json, speed, cost_usd_raw,
|
|
2613
|
-
mutation_seq, mutation_min_ts)
|
|
2614
|
-
VALUES (
|
|
2779
|
+
mutation_seq, mutation_min_ts, account_key)
|
|
2780
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
2615
2781
|
ON CONFLICT(msg_id, req_id)
|
|
2616
2782
|
WHERE msg_id IS NOT NULL AND req_id IS NOT NULL
|
|
2617
2783
|
DO UPDATE SET
|
|
@@ -2696,20 +2862,28 @@ def sync_cache(
|
|
|
2696
2862
|
# UPSERT preserves session_id / project_path columns populated
|
|
2697
2863
|
# by _ensure_session_files_row at the top of this loop. A plain
|
|
2698
2864
|
# INSERT OR REPLACE would wipe them on every changed-file sync.
|
|
2865
|
+
# #341: session_files.account_key is a "last observed" diagnostic
|
|
2866
|
+
# only (spec §2 — entry rows are authoritative for attribution; no
|
|
2867
|
+
# aggregation reads file rows). Stamp the active account resolved
|
|
2868
|
+
# once per sync; None (stably-absent) writes NULL.
|
|
2699
2869
|
conn.execute(
|
|
2700
2870
|
"""INSERT INTO session_files
|
|
2701
|
-
(path, size_bytes, mtime_ns, last_byte_offset,
|
|
2702
|
-
|
|
2871
|
+
(path, size_bytes, mtime_ns, last_byte_offset,
|
|
2872
|
+
last_ingested_at, account_key)
|
|
2873
|
+
VALUES (?,?,?,?,?,?)
|
|
2703
2874
|
ON CONFLICT(path) DO UPDATE SET
|
|
2704
2875
|
size_bytes = excluded.size_bytes,
|
|
2705
2876
|
mtime_ns = excluded.mtime_ns,
|
|
2706
2877
|
last_byte_offset = excluded.last_byte_offset,
|
|
2707
|
-
last_ingested_at = excluded.last_ingested_at
|
|
2878
|
+
last_ingested_at = excluded.last_ingested_at,
|
|
2879
|
+
account_key = excluded.account_key""",
|
|
2708
2880
|
(
|
|
2709
2881
|
path_str, size, mtime_ns, final_offset,
|
|
2710
2882
|
dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
2883
|
+
file_account_key,
|
|
2711
2884
|
),
|
|
2712
2885
|
)
|
|
2886
|
+
_cache_storm_test_pause("claude_precommit")
|
|
2713
2887
|
conn.commit()
|
|
2714
2888
|
stats.files_processed += 1
|
|
2715
2889
|
# Browse-rail rollup: record the session_ids this file just
|
|
@@ -2721,6 +2895,12 @@ def sync_cache(
|
|
|
2721
2895
|
except sqlite3.DatabaseError as exc:
|
|
2722
2896
|
eprint(f"[cache] db error on {jp}: {exc}")
|
|
2723
2897
|
conn.rollback()
|
|
2898
|
+
if _cctally_db_sib._is_sqlite_corruption_error(exc):
|
|
2899
|
+
# Corruption is a database-family failure, not a bad input
|
|
2900
|
+
# file. The recovery plan must see it immediately so it can
|
|
2901
|
+
# close this handle, quarantine once, and restart the full
|
|
2902
|
+
# requested provider plan on a fresh family.
|
|
2903
|
+
raise
|
|
2724
2904
|
walk_clean = False # rolled back this file without ingesting (D5a)
|
|
2725
2905
|
stats.files_failed += 1
|
|
2726
2906
|
continue
|
|
@@ -3464,12 +3644,19 @@ def iter_entries(
|
|
|
3464
3644
|
range_end: dt.datetime,
|
|
3465
3645
|
*,
|
|
3466
3646
|
project: str | None = None,
|
|
3647
|
+
account_key: "str | None" = None,
|
|
3467
3648
|
) -> list[UsageEntry]:
|
|
3468
3649
|
"""Return cached UsageEntry rows whose timestamp falls in [range_start,
|
|
3469
3650
|
range_end]. Optional `project` filters by the project slug (directory
|
|
3470
3651
|
name under `<claude>/projects/`). Drop-in replacement for the old
|
|
3471
3652
|
`_discover_session_files` + `_parse_usage_entries` loop; dedup is
|
|
3472
3653
|
enforced at write time by the UNIQUE(msg_id, req_id) index.
|
|
3654
|
+
|
|
3655
|
+
``account_key`` (#341, P2-CQ2) scopes the sum to one account's stamped
|
|
3656
|
+
entries. ``None`` (default) reads all accounts (merged, byte-identical to
|
|
3657
|
+
today). The reserved ``unattributed`` sentinel matches BOTH the literal
|
|
3658
|
+
stamp AND a NULL ``account_key`` (read rule: ``NULL ≡ unattributed``), so a
|
|
3659
|
+
single-account / legacy install whose rows are all NULL sums identically.
|
|
3473
3660
|
"""
|
|
3474
3661
|
start_iso = range_start.astimezone(dt.timezone.utc).isoformat()
|
|
3475
3662
|
end_iso = range_end.astimezone(dt.timezone.utc).isoformat()
|
|
@@ -3482,6 +3669,14 @@ def iter_entries(
|
|
|
3482
3669
|
"WHERE timestamp_utc >= ? AND timestamp_utc <= ?"
|
|
3483
3670
|
)
|
|
3484
3671
|
params: list[Any] = [start_iso, end_iso]
|
|
3672
|
+
if account_key is not None:
|
|
3673
|
+
import _lib_accounts
|
|
3674
|
+
if account_key == _lib_accounts.UNATTRIBUTED:
|
|
3675
|
+
sql += " AND (account_key IS NULL OR account_key = ?)"
|
|
3676
|
+
params.append(_lib_accounts.UNATTRIBUTED)
|
|
3677
|
+
else:
|
|
3678
|
+
sql += " AND account_key = ?"
|
|
3679
|
+
params.append(account_key)
|
|
3485
3680
|
if project is not None:
|
|
3486
3681
|
# Escape LIKE wildcards (_ matches any single char, % matches any
|
|
3487
3682
|
# string). The old glob-based discovery matched project names
|
|
@@ -3593,6 +3788,25 @@ def iter_entries_with_id(
|
|
|
3593
3788
|
return out
|
|
3594
3789
|
|
|
3595
3790
|
|
|
3791
|
+
class AccountAttributionUnavailable(Exception):
|
|
3792
|
+
"""A ``--account``-scoped entry read cannot be satisfied from the account-
|
|
3793
|
+
stamped cache and would otherwise degrade to a direct-JSONL parse (#341,
|
|
3794
|
+
spec §3). The historical JSONL carries NO account identity, so that fallback
|
|
3795
|
+
would silently return unfiltered/merged (or empty) entries mislabeled as the
|
|
3796
|
+
selected account. The CLI maps this to exit 3
|
|
3797
|
+
(``account attribution unavailable (cache required)``) — failing closed
|
|
3798
|
+
rather than emitting an unattributable render. Only raised when the caller
|
|
3799
|
+
passed a real ``account_key`` (merged reads keep the correctness-degrade)."""
|
|
3800
|
+
|
|
3801
|
+
|
|
3802
|
+
def _guard_account_attribution(account_key: "str | None", where: str) -> None:
|
|
3803
|
+
"""Fail closed (#341) when an ``account_key``-scoped read would fall back to
|
|
3804
|
+
the identity-less direct-JSONL path. No-op for merged (``None``) reads."""
|
|
3805
|
+
if account_key is not None:
|
|
3806
|
+
raise AccountAttributionUnavailable(
|
|
3807
|
+
f"account attribution unavailable (cache required): {where}")
|
|
3808
|
+
|
|
3809
|
+
|
|
3596
3810
|
def _collect_entries_direct(
|
|
3597
3811
|
range_start: dt.datetime,
|
|
3598
3812
|
range_end: dt.datetime,
|
|
@@ -3664,6 +3878,7 @@ def get_claude_session_entries(
|
|
|
3664
3878
|
*,
|
|
3665
3879
|
project: str | None = None,
|
|
3666
3880
|
skip_sync: bool = False,
|
|
3881
|
+
account_key: "str | None" = None,
|
|
3667
3882
|
) -> list[_JoinedClaudeEntry]:
|
|
3668
3883
|
"""Fetch in-range Claude entries joined to per-file metadata.
|
|
3669
3884
|
|
|
@@ -3686,17 +3901,25 @@ def get_claude_session_entries(
|
|
|
3686
3901
|
try:
|
|
3687
3902
|
conn = open_cache_db()
|
|
3688
3903
|
except (sqlite3.DatabaseError, OSError) as exc:
|
|
3904
|
+
# #341: an account-scoped read can't degrade to the identity-less
|
|
3905
|
+
# direct-JSONL path — fail closed (exit 3) rather than mislabel.
|
|
3906
|
+
_guard_account_attribution(account_key, "cache open failed")
|
|
3689
3907
|
eprint(f"[cache] unavailable ({exc}); falling back to direct JSONL parse")
|
|
3690
3908
|
return _direct_parse_claude_session_entries(
|
|
3691
3909
|
range_start, range_end, project=project
|
|
3692
3910
|
)
|
|
3693
3911
|
|
|
3694
3912
|
if not skip_sync:
|
|
3695
|
-
stats =
|
|
3913
|
+
stats, conn = _run_cache_operation_with_recovery(
|
|
3914
|
+
conn, lambda active_conn: sync_cache(active_conn)
|
|
3915
|
+
)
|
|
3696
3916
|
if stats.lock_contended:
|
|
3697
3917
|
# Partial cache window: a concurrent ingest may have committed some
|
|
3698
3918
|
# files but not others. For correctness, fall back to a direct
|
|
3699
3919
|
# JSONL parse — same rationale as `get_entries`.
|
|
3920
|
+
# #341: fail closed on an account-scoped read — the direct-JSONL
|
|
3921
|
+
# fallback carries no account identity (exit 3, not a mislabel).
|
|
3922
|
+
_guard_account_attribution(account_key, "concurrent ingest")
|
|
3700
3923
|
eprint(
|
|
3701
3924
|
"[cache] concurrent ingest in progress; "
|
|
3702
3925
|
"falling back to direct JSONL parse for correctness"
|
|
@@ -3727,6 +3950,17 @@ def get_claude_session_entries(
|
|
|
3727
3950
|
)
|
|
3728
3951
|
sql += r" AND se.source_path LIKE ? ESCAPE '\'"
|
|
3729
3952
|
params.append(f"%/projects/{escaped}/%")
|
|
3953
|
+
if account_key is not None:
|
|
3954
|
+
# #341 --account scoping (mirrors iter_entries/get_entries). The reserved
|
|
3955
|
+
# ``unattributed`` sentinel matches BOTH the literal stamp AND NULL
|
|
3956
|
+
# (``NULL ≡ unattributed`` on the read path); a real key matches exactly.
|
|
3957
|
+
import _lib_accounts
|
|
3958
|
+
if account_key == _lib_accounts.UNATTRIBUTED:
|
|
3959
|
+
sql += " AND (se.account_key IS NULL OR se.account_key = ?)"
|
|
3960
|
+
params.append(_lib_accounts.UNATTRIBUTED)
|
|
3961
|
+
else:
|
|
3962
|
+
sql += " AND se.account_key = ?"
|
|
3963
|
+
params.append(account_key)
|
|
3730
3964
|
# Explicit (timestamp_utc, id) tie-break (#275) — the same contract #271 §5
|
|
3731
3965
|
# pinned on `get_entries` (see the twin ORDER BY above). `id` is the rowid, so
|
|
3732
3966
|
# against `idx_entries_timestamp` (which stores keys as (timestamp_utc, rowid))
|
|
@@ -3935,6 +4169,10 @@ class CodexIngestStats:
|
|
|
3935
4169
|
# normalization, DB exception).
|
|
3936
4170
|
files_failed: int = 0
|
|
3937
4171
|
deferred_reason: "str | None" = None
|
|
4172
|
+
# #341: files whose root auth.json read was torn (mid-rewrite) this cycle.
|
|
4173
|
+
# Deferred WITHOUT advancing their cursor so the next sync re-reads and
|
|
4174
|
+
# re-stamps rather than guessing an account (spec §1 stable-read protocol).
|
|
4175
|
+
files_deferred_torn: int = 0
|
|
3938
4176
|
|
|
3939
4177
|
@property
|
|
3940
4178
|
def targeted_clean(self) -> bool:
|
|
@@ -3968,9 +4206,10 @@ def sync_codex_cache(
|
|
|
3968
4206
|
) -> CodexIngestStats:
|
|
3969
4207
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
3970
4208
|
|
|
3971
|
-
Acquires
|
|
3972
|
-
|
|
3973
|
-
|
|
4209
|
+
Acquires the shared cache.db writer flock before the Codex provider flock.
|
|
4210
|
+
The global-first order excludes cross-provider writes and checkpoints while
|
|
4211
|
+
preserving provider-scoped migration/rebuild coordination. On contention
|
|
4212
|
+
returns immediately with lock_contended=True.
|
|
3974
4213
|
|
|
3975
4214
|
When `rebuild=True`, clears the cached rows AFTER acquiring the lock
|
|
3976
4215
|
so a lost race does not wipe a cache another process is actively
|
|
@@ -3990,15 +4229,24 @@ def sync_codex_cache(
|
|
|
3990
4229
|
deferred_cert_roots: "set[str] | None" = None
|
|
3991
4230
|
deferred_cert_sigs: "dict[str, str] | None" = None
|
|
3992
4231
|
c = _cctally()
|
|
3993
|
-
|
|
3994
|
-
|
|
4232
|
+
from _lib_cache_writer_lock import (
|
|
4233
|
+
acquire_cache_writer_flocks,
|
|
4234
|
+
release_cache_writer_flocks,
|
|
4235
|
+
)
|
|
3995
4236
|
|
|
3996
|
-
|
|
4237
|
+
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
4238
|
+
held_writer_flocks: list[int] = []
|
|
3997
4239
|
try:
|
|
3998
4240
|
with _perf.phase("flock") as _p_flock:
|
|
3999
|
-
|
|
4000
|
-
|
|
4001
|
-
|
|
4241
|
+
acquired = acquire_cache_writer_flocks(
|
|
4242
|
+
_cctally_core.CACHE_LOCK_PATH,
|
|
4243
|
+
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
4244
|
+
timeout=lock_timeout,
|
|
4245
|
+
)
|
|
4246
|
+
if acquired is not None:
|
|
4247
|
+
held_writer_flocks = acquired
|
|
4248
|
+
_p_flock.set_meta(contended=acquired is None)
|
|
4249
|
+
if acquired is None:
|
|
4002
4250
|
eprint("[codex-cache] sync already in progress; using existing cache")
|
|
4003
4251
|
stats.lock_contended = True
|
|
4004
4252
|
return stats
|
|
@@ -4161,6 +4409,10 @@ def sync_codex_cache(
|
|
|
4161
4409
|
stats.deferred_reason = "truncation"
|
|
4162
4410
|
return stats
|
|
4163
4411
|
|
|
4412
|
+
# #341: per-root active-account cache, resolved once per sync (auth.json
|
|
4413
|
+
# is per provider root and rarely changes mid-sync). Keyed by
|
|
4414
|
+
# source_root_key. A torn read defers every file under that root.
|
|
4415
|
+
root_accounts: "dict[str, _CodexRootAccount]" = {}
|
|
4164
4416
|
# #279 S2 F4: ONE coarse `walk` phase bracketing the per-file loop
|
|
4165
4417
|
# (count = files_processed, never per-row — §2 rule). Manual CM so
|
|
4166
4418
|
# the loop stays flat, mirroring sync_cache's walk seam.
|
|
@@ -4230,6 +4482,30 @@ def sync_codex_cache(
|
|
|
4230
4482
|
initial_total_tokens = 0
|
|
4231
4483
|
prev_total_tokens = None
|
|
4232
4484
|
|
|
4485
|
+
# #341: resolve this root's active account (per-root auth.json
|
|
4486
|
+
# stable-read, resolved once per sync). A torn read (auth.json
|
|
4487
|
+
# mid-rewrite) DEFERS the whole file this cycle — skip its new bytes
|
|
4488
|
+
# WITHOUT advancing the cursor, so the next sync re-reads and
|
|
4489
|
+
# re-stamps rather than guessing an account (spec §1 stable-read
|
|
4490
|
+
# protocol). identified -> real key; stably-absent (no auth /
|
|
4491
|
+
# api-key mode) -> None (stamped NULL == unattributed on read).
|
|
4492
|
+
root_account = root_accounts.get(discovered.source_root_key)
|
|
4493
|
+
if root_account is None:
|
|
4494
|
+
root_account = _resolve_codex_account_for_root(
|
|
4495
|
+
discovered.provider_root)
|
|
4496
|
+
root_accounts[discovered.source_root_key] = root_account
|
|
4497
|
+
if root_account.status == "torn":
|
|
4498
|
+
stats.files_deferred_torn += 1
|
|
4499
|
+
if targeted:
|
|
4500
|
+
stats.files_failed += 1 # §5.1 deferred → call dirty
|
|
4501
|
+
continue
|
|
4502
|
+
file_account_key = root_account.account_key
|
|
4503
|
+
# First-sight registry observe, journaled DURABLY BEFORE any
|
|
4504
|
+
# account-stamped quota obs / cache row for this account (spec §1:
|
|
4505
|
+
# replay can never see a stamped row whose account was never
|
|
4506
|
+
# observed). Marker-deduped; no-op for the sentinel.
|
|
4507
|
+
_maybe_append_codex_account_observe(root_account.identity)
|
|
4508
|
+
|
|
4233
4509
|
accounting_rows: list[tuple[Any, ...]] = []
|
|
4234
4510
|
quota_rows: list[tuple[Any, ...]] = []
|
|
4235
4511
|
thread_rows: list[tuple[Any, ...]] = []
|
|
@@ -4296,6 +4572,7 @@ def sync_codex_cache(
|
|
|
4296
4572
|
quota.used_percent, quota.resets_at_utc,
|
|
4297
4573
|
quota.plan_type, quota.individual_limit_json,
|
|
4298
4574
|
quota.reached_type, iter_state.model,
|
|
4575
|
+
file_account_key, # #341 trailing account_key
|
|
4299
4576
|
))
|
|
4300
4577
|
if (thread := emission.thread) is not None and (
|
|
4301
4578
|
thread.conversation_key is not None
|
|
@@ -4325,6 +4602,7 @@ def sync_codex_cache(
|
|
|
4325
4602
|
entry.total_tokens,
|
|
4326
4603
|
discovered.source_root_key,
|
|
4327
4604
|
event.conversation_key,
|
|
4605
|
+
file_account_key, # #341 trailing account_key
|
|
4328
4606
|
))
|
|
4329
4607
|
yielded_count += 1
|
|
4330
4608
|
final_offset = fh.tell()
|
|
@@ -4430,9 +4708,16 @@ def sync_codex_cache(
|
|
|
4430
4708
|
# §5.1 whole-tree bypass: targeted mode never prunes
|
|
4431
4709
|
# codex_source_roots for roots outside its target set.
|
|
4432
4710
|
prune_roots=not targeted,
|
|
4711
|
+
account_key=file_account_key, # #341 last-observed stamp
|
|
4433
4712
|
)
|
|
4434
4713
|
except sqlite3.DatabaseError as exc:
|
|
4435
4714
|
conn.rollback()
|
|
4715
|
+
if _cctally_db_sib._is_sqlite_corruption_error(exc):
|
|
4716
|
+
# Retrying a corrupt SQLite connection in place cannot
|
|
4717
|
+
# heal it and hides the signal from the shared recovery
|
|
4718
|
+
# boundary. Propagate classified family corruption on
|
|
4719
|
+
# the first observation.
|
|
4720
|
+
raise
|
|
4436
4721
|
if attempt == 0:
|
|
4437
4722
|
# Private test seam: the callback runs after the
|
|
4438
4723
|
# failed file transaction has rolled back, and before
|
|
@@ -4449,8 +4734,10 @@ def sync_codex_cache(
|
|
|
4449
4734
|
committed = True
|
|
4450
4735
|
break
|
|
4451
4736
|
if not committed:
|
|
4452
|
-
|
|
4453
|
-
|
|
4737
|
+
# Both whole-tree and targeted callers need an honest failed
|
|
4738
|
+
# walk count. Targeted mode already depended on this signal;
|
|
4739
|
+
# explicit rebuild now uses it to reject partial success too.
|
|
4740
|
+
stats.files_failed += 1
|
|
4454
4741
|
continue
|
|
4455
4742
|
|
|
4456
4743
|
# Private test seam (§5.1 post-preflight late-shrink race): fires
|
|
@@ -4469,8 +4756,8 @@ def sync_codex_cache(
|
|
|
4469
4756
|
_p_walk.set_meta(skipped=stats.files_skipped_unchanged,
|
|
4470
4757
|
rows=stats.rows_changed)
|
|
4471
4758
|
# #279 S2 F1: rolling parse-health record (codex half). Same
|
|
4472
|
-
# anomaly-delta gate as the Claude tail;
|
|
4473
|
-
#
|
|
4759
|
+
# anomaly-delta gate as the Claude tail; the global writer flock
|
|
4760
|
+
# excludes a concurrent Claude sync.
|
|
4474
4761
|
_update_parse_health_meta(
|
|
4475
4762
|
conn, "parse_health_codex",
|
|
4476
4763
|
lines_seen=stats.lines_seen,
|
|
@@ -4480,14 +4767,14 @@ def sync_codex_cache(
|
|
|
4480
4767
|
rebuild=rebuild,
|
|
4481
4768
|
)
|
|
4482
4769
|
# Codex creates/extends cache.db sidecars independently of Claude's
|
|
4483
|
-
# sync path. Harden them while
|
|
4484
|
-
# all Codex writes, before the optional checkpoint can rotate a
|
|
4770
|
+
# sync path. Harden them while both cache flocks are still held and
|
|
4771
|
+
# after all Codex writes, before the optional checkpoint can rotate a
|
|
4772
|
+
# WAL.
|
|
4485
4773
|
_harden_cache_sidecars()
|
|
4486
|
-
# #297: forced end-of-sync WAL drain (Codex half).
|
|
4487
|
-
#
|
|
4488
|
-
#
|
|
4489
|
-
#
|
|
4490
|
-
# on. All Codex ingest work is committed here (no active txn).
|
|
4774
|
+
# #297/#344: forced end-of-sync WAL drain (Codex half). The global
|
|
4775
|
+
# writer flock excludes every Claude write/checkpoint until this
|
|
4776
|
+
# checkpoint finishes. All Codex ingest work is committed here (no
|
|
4777
|
+
# active transaction).
|
|
4491
4778
|
_maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
|
|
4492
4779
|
# Projection intentionally runs only after this function releases the
|
|
4493
4780
|
# Codex cache flock in ``finally`` below. cache.db and stats.db are not
|
|
@@ -4549,11 +4836,7 @@ def sync_codex_cache(
|
|
|
4549
4836
|
else:
|
|
4550
4837
|
project_after_unlock = True
|
|
4551
4838
|
finally:
|
|
4552
|
-
|
|
4553
|
-
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4554
|
-
except OSError:
|
|
4555
|
-
pass
|
|
4556
|
-
lock_fh.close()
|
|
4839
|
+
release_cache_writer_flocks(held_writer_flocks)
|
|
4557
4840
|
|
|
4558
4841
|
if deferred_cert_roots is not None:
|
|
4559
4842
|
# F1: the gate's skip-condition must be IDENTICAL to
|
|
@@ -4583,8 +4866,16 @@ def iter_codex_entries(
|
|
|
4583
4866
|
conn: sqlite3.Connection,
|
|
4584
4867
|
range_start: dt.datetime,
|
|
4585
4868
|
range_end: dt.datetime,
|
|
4869
|
+
*,
|
|
4870
|
+
account_key: "str | None" = None,
|
|
4586
4871
|
) -> list[CodexEntry]:
|
|
4587
|
-
"""Return cached CodexEntry rows with timestamp in [range_start, range_end].
|
|
4872
|
+
"""Return cached CodexEntry rows with timestamp in [range_start, range_end].
|
|
4873
|
+
|
|
4874
|
+
``account_key`` (#341 Step 4-eval) scopes the read to one Codex account's
|
|
4875
|
+
stamped entries for the per-account budget ladder; ``None`` (default) reads
|
|
4876
|
+
all accounts (merged / byte-identical). The ``unattributed`` sentinel matches
|
|
4877
|
+
the literal stamp OR a NULL ``account_key`` (read rule ``NULL ≡ unattributed``).
|
|
4878
|
+
"""
|
|
4588
4879
|
start_iso = range_start.astimezone(dt.timezone.utc).isoformat()
|
|
4589
4880
|
end_iso = range_end.astimezone(dt.timezone.utc).isoformat()
|
|
4590
4881
|
sql = (
|
|
@@ -4593,10 +4884,19 @@ def iter_codex_entries(
|
|
|
4593
4884
|
"reasoning_output_tokens, total_tokens, source_path "
|
|
4594
4885
|
"FROM codex_session_entries "
|
|
4595
4886
|
"WHERE timestamp_utc >= ? AND timestamp_utc <= ? "
|
|
4596
|
-
"ORDER BY timestamp_utc ASC"
|
|
4597
4887
|
)
|
|
4888
|
+
params: "list[Any]" = [start_iso, end_iso]
|
|
4889
|
+
if account_key is not None:
|
|
4890
|
+
import _lib_accounts
|
|
4891
|
+
if account_key == _lib_accounts.UNATTRIBUTED:
|
|
4892
|
+
sql += "AND (account_key IS NULL OR account_key = ?) "
|
|
4893
|
+
params.append(_lib_accounts.UNATTRIBUTED)
|
|
4894
|
+
else:
|
|
4895
|
+
sql += "AND account_key = ? "
|
|
4896
|
+
params.append(account_key)
|
|
4897
|
+
sql += "ORDER BY timestamp_utc ASC"
|
|
4598
4898
|
entries: list[CodexEntry] = []
|
|
4599
|
-
for row in conn.execute(sql,
|
|
4899
|
+
for row in conn.execute(sql, params):
|
|
4600
4900
|
entries.append(CodexEntry(
|
|
4601
4901
|
timestamp=dt.datetime.fromisoformat(row[0]),
|
|
4602
4902
|
session_id=row[1],
|
|
@@ -4635,6 +4935,7 @@ def get_codex_entries(
|
|
|
4635
4935
|
range_end: dt.datetime,
|
|
4636
4936
|
*,
|
|
4637
4937
|
skip_sync: bool = False,
|
|
4938
|
+
account_key: "str | None" = None,
|
|
4638
4939
|
) -> list[CodexEntry]:
|
|
4639
4940
|
"""Cache-first Codex entry fetch with transparent fallback.
|
|
4640
4941
|
|
|
@@ -4650,12 +4951,23 @@ def get_codex_entries(
|
|
|
4650
4951
|
try:
|
|
4651
4952
|
conn = open_cache_db()
|
|
4652
4953
|
except (sqlite3.DatabaseError, OSError) as exc:
|
|
4954
|
+
# #341 (Task 4, fail-closed symmetry with the Claude siblings): an
|
|
4955
|
+
# account-scoped read can't degrade to the identity-less direct-JSONL
|
|
4956
|
+
# path — fail closed (exit 3) rather than return ALL Codex entries
|
|
4957
|
+
# mislabeled as the selected account. No-op for merged (None) reads.
|
|
4958
|
+
_guard_account_attribution(account_key, "cache open failed")
|
|
4653
4959
|
eprint(f"[cache] unavailable ({exc}); falling back to direct JSONL parse")
|
|
4654
4960
|
return _collect_codex_entries_direct(range_start, range_end)
|
|
4655
4961
|
try:
|
|
4656
4962
|
if skip_sync:
|
|
4657
|
-
return iter_codex_entries(
|
|
4658
|
-
|
|
4963
|
+
return iter_codex_entries(
|
|
4964
|
+
conn, range_start, range_end, account_key=account_key)
|
|
4965
|
+
# #344: route the ingest through the guarded recovery boundary so a
|
|
4966
|
+
# classified corruption closes the handle, quarantines once, and
|
|
4967
|
+
# restarts on a fresh family (reassigning `conn`, closed in finally).
|
|
4968
|
+
stats, conn = _run_cache_operation_with_recovery(
|
|
4969
|
+
conn, lambda active_conn: sync_codex_cache(active_conn)
|
|
4970
|
+
)
|
|
4659
4971
|
if stats.lock_contended:
|
|
4660
4972
|
# Sync commits file-by-file, so contention on the ingest lock
|
|
4661
4973
|
# (e.g. a concurrent --rebuild, or a first-run sync still in
|
|
@@ -4665,12 +4977,17 @@ def get_codex_entries(
|
|
|
4665
4977
|
# caller's range. Fall back to a direct JSONL parse unconditionally
|
|
4666
4978
|
# on contention; correctness > speed in the rare-but-real window
|
|
4667
4979
|
# where cache state does not match disk.
|
|
4980
|
+
# #341 (Task 4): fail closed on an account-scoped read — the
|
|
4981
|
+
# direct-JSONL fallback carries no account identity (exit 3, not a
|
|
4982
|
+
# mislabel). Symmetric with get_entries / get_claude_session_entries.
|
|
4983
|
+
_guard_account_attribution(account_key, "concurrent ingest")
|
|
4668
4984
|
eprint(
|
|
4669
4985
|
"[cache] concurrent codex ingest in progress; "
|
|
4670
4986
|
"falling back to direct JSONL parse for correctness"
|
|
4671
4987
|
)
|
|
4672
4988
|
return _collect_codex_entries_direct(range_start, range_end)
|
|
4673
|
-
return iter_codex_entries(
|
|
4989
|
+
return iter_codex_entries(
|
|
4990
|
+
conn, range_start, range_end, account_key=account_key)
|
|
4674
4991
|
finally:
|
|
4675
4992
|
conn.close()
|
|
4676
4993
|
|
|
@@ -4681,6 +4998,7 @@ def _sum_codex_cost_for_range(
|
|
|
4681
4998
|
*,
|
|
4682
4999
|
speed: str = "auto",
|
|
4683
5000
|
skip_sync: bool = False,
|
|
5001
|
+
account_key: "str | None" = None,
|
|
4684
5002
|
) -> float:
|
|
4685
5003
|
"""Sum USD Codex cost of all `codex_session_entries` in ``[start, end)``.
|
|
4686
5004
|
|
|
@@ -4715,7 +5033,8 @@ def _sum_codex_cost_for_range(
|
|
|
4715
5033
|
c = _cctally()
|
|
4716
5034
|
eff_speed = c._resolve_codex_speed(speed)
|
|
4717
5035
|
total = 0.0
|
|
4718
|
-
for entry in c.get_codex_entries(start, end, skip_sync=skip_sync
|
|
5036
|
+
for entry in c.get_codex_entries(start, end, skip_sync=skip_sync,
|
|
5037
|
+
account_key=account_key):
|
|
4719
5038
|
if entry.timestamp >= end:
|
|
4720
5039
|
continue
|
|
4721
5040
|
total += c._calculate_codex_entry_cost(
|
|
@@ -4735,6 +5054,7 @@ def get_entries(
|
|
|
4735
5054
|
*,
|
|
4736
5055
|
project: str | None = None,
|
|
4737
5056
|
skip_sync: bool = False,
|
|
5057
|
+
account_key: "str | None" = None,
|
|
4738
5058
|
) -> list[UsageEntry]:
|
|
4739
5059
|
"""Cache-first entry fetch with transparent fallback. Every JSONL-consuming
|
|
4740
5060
|
command should use this instead of talking to open_cache_db directly.
|
|
@@ -4742,29 +5062,56 @@ def get_entries(
|
|
|
4742
5062
|
When `skip_sync=True`, bypass the JSONL ingest and serve whatever is
|
|
4743
5063
|
already cached. The cache-open fallback still fires if the cache DB is
|
|
4744
5064
|
unusable, but the ingest + lock-contention fallback are both skipped.
|
|
5065
|
+
|
|
5066
|
+
``account_key`` (#341, P2-CQ2) scopes the cache read to one account's
|
|
5067
|
+
stamped entries (``None`` = merged / byte-identical). Not threaded into the
|
|
5068
|
+
direct-JSONL fallback: historical JSONL lines carry no identity, so an
|
|
5069
|
+
account-scoped read that has to fall back is an unattributable degrade — the
|
|
5070
|
+
``--account`` CLI surface fails closed (exit 3) before it reaches here.
|
|
4745
5071
|
"""
|
|
4746
5072
|
try:
|
|
4747
5073
|
conn = open_cache_db()
|
|
4748
5074
|
except (sqlite3.DatabaseError, OSError) as exc:
|
|
5075
|
+
# #341: an account-scoped read can't degrade to the identity-less
|
|
5076
|
+
# direct-JSONL path — fail closed (exit 3) rather than mislabel.
|
|
5077
|
+
_guard_account_attribution(account_key, "cache open failed")
|
|
4749
5078
|
eprint(f"[cache] unavailable ({exc}); falling back to direct JSONL parse")
|
|
4750
5079
|
return _collect_entries_direct(range_start, range_end, project=project)
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
|
|
4754
|
-
|
|
4755
|
-
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
# where cache state does not match disk.
|
|
4762
|
-
eprint(
|
|
4763
|
-
"[cache] concurrent ingest in progress; "
|
|
4764
|
-
"falling back to direct JSONL parse for correctness"
|
|
5080
|
+
# Close the cache connection on every return path (#341 P2-CQ2 hygiene): the
|
|
5081
|
+
# prior code opened it and returned `iter_entries(conn, …)` — a materialized
|
|
5082
|
+
# list — without ever closing, leaking a connection per call (ResourceWarning
|
|
5083
|
+
# at GC). iter_entries fully materializes, so closing after is byte-neutral.
|
|
5084
|
+
# The finally also closes the #344 recovery-replacement `conn` (a classified
|
|
5085
|
+
# corruption reassigns it to a fresh family here); double-close is a no-op.
|
|
5086
|
+
try:
|
|
5087
|
+
if not skip_sync:
|
|
5088
|
+
stats, conn = _run_cache_operation_with_recovery(
|
|
5089
|
+
conn, lambda active_conn: sync_cache(active_conn)
|
|
4765
5090
|
)
|
|
4766
|
-
|
|
4767
|
-
|
|
5091
|
+
if stats.lock_contended:
|
|
5092
|
+
# Sync commits file-by-file, so contention on the ingest lock
|
|
5093
|
+
# (e.g. a concurrent --rebuild, or a first-run sync still in
|
|
5094
|
+
# flight) can leave the cache PARTIALLY populated — some files
|
|
5095
|
+
# ingested, others pending. An "is the table empty?" guard passes
|
|
5096
|
+
# in that window and we'd silently return results missing the
|
|
5097
|
+
# caller's range. Fall back to a direct JSONL parse unconditionally
|
|
5098
|
+
# on contention; correctness > speed in the rare-but-real window
|
|
5099
|
+
# where cache state does not match disk.
|
|
5100
|
+
# #341: fail closed on an account-scoped read — the direct-JSONL
|
|
5101
|
+
# fallback carries no account identity (exit 3, not a mislabel).
|
|
5102
|
+
_guard_account_attribution(account_key, "concurrent ingest")
|
|
5103
|
+
eprint(
|
|
5104
|
+
"[cache] concurrent ingest in progress; "
|
|
5105
|
+
"falling back to direct JSONL parse for correctness"
|
|
5106
|
+
)
|
|
5107
|
+
return _collect_entries_direct(range_start, range_end, project=project)
|
|
5108
|
+
return iter_entries(
|
|
5109
|
+
conn, range_start, range_end, project=project, account_key=account_key)
|
|
5110
|
+
finally:
|
|
5111
|
+
try:
|
|
5112
|
+
conn.close()
|
|
5113
|
+
except Exception:
|
|
5114
|
+
pass
|
|
4768
5115
|
|
|
4769
5116
|
|
|
4770
5117
|
def _harden_cache_sidecars() -> None:
|
|
@@ -4798,52 +5145,104 @@ def _cache_open_guarded() -> sqlite3.Connection:
|
|
|
4798
5145
|
"""
|
|
4799
5146
|
path = pathlib.Path(_cctally_core.CACHE_DB_PATH)
|
|
4800
5147
|
marker = _cctally_db_sib._repair_marker_path(path)
|
|
5148
|
+
pending = _cctally_db_sib._quarantine_pending_path(path)
|
|
4801
5149
|
lock_path = pathlib.Path(_cctally_core.CACHE_LOCK_MAINTENANCE_PATH)
|
|
4802
5150
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
4803
5151
|
lock_fh = open(lock_path, "a+")
|
|
4804
|
-
conn = None
|
|
4805
5152
|
try:
|
|
4806
|
-
|
|
4807
|
-
if marker.exists():
|
|
4808
|
-
raise sqlite3.DatabaseError(
|
|
4809
|
-
f"cache.db maintenance is in progress ({marker})"
|
|
4810
|
-
)
|
|
4811
|
-
conn = _cctally_store.open_index("cache")
|
|
4812
|
-
# Force a real header/schema read. SELECT 1 is constant-folded and can
|
|
4813
|
-
# report success without touching a malformed database file.
|
|
4814
|
-
conn.execute("PRAGMA schema_version").fetchone()
|
|
4815
|
-
# The v1.80.2 production incident left the schema and left edge of
|
|
4816
|
-
# session_entries readable while its interior root page pointed past
|
|
4817
|
-
# EOF on the right. Probe that right-most path explicitly: O(log N), so
|
|
4818
|
-
# every short-lived hook can afford it, unlike PRAGMA quick_check's
|
|
4819
|
-
# whole-database scan.
|
|
4820
|
-
if conn.execute(
|
|
4821
|
-
"SELECT 1 FROM sqlite_schema "
|
|
4822
|
-
"WHERE type='table' AND name='session_entries'"
|
|
4823
|
-
).fetchone() is not None:
|
|
4824
|
-
conn.execute(
|
|
4825
|
-
"SELECT rowid FROM session_entries "
|
|
4826
|
-
"ORDER BY rowid DESC LIMIT 1"
|
|
4827
|
-
).fetchone()
|
|
4828
|
-
if marker.exists():
|
|
4829
|
-
conn.close()
|
|
5153
|
+
for _attempt in range(2):
|
|
4830
5154
|
conn = None
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
5155
|
+
fcntl.flock(lock_fh, fcntl.LOCK_SH)
|
|
5156
|
+
if marker.exists() or pending.exists():
|
|
5157
|
+
live, reason = (
|
|
5158
|
+
_cctally_db_sib._repair_marker_is_live(marker)
|
|
5159
|
+
if marker.exists()
|
|
5160
|
+
else (False, "pending quarantine")
|
|
5161
|
+
)
|
|
5162
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
5163
|
+
if live:
|
|
5164
|
+
raise sqlite3.DatabaseError(
|
|
5165
|
+
f"cache.db maintenance is in progress ({reason})"
|
|
5166
|
+
)
|
|
5167
|
+
# Drop shared before taking exclusive so two stale-marker
|
|
5168
|
+
# reclaimers cannot deadlock while upgrading. Recheck under
|
|
5169
|
+
# exclusive: a live owner may have replaced the stale marker.
|
|
5170
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX)
|
|
5171
|
+
if marker.exists():
|
|
5172
|
+
live, reason = _cctally_db_sib._repair_marker_is_live(marker)
|
|
5173
|
+
if live:
|
|
5174
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
5175
|
+
raise sqlite3.DatabaseError(
|
|
5176
|
+
f"cache.db maintenance is in progress ({reason})"
|
|
5177
|
+
)
|
|
5178
|
+
if pending.exists():
|
|
5179
|
+
try:
|
|
5180
|
+
open_pids = _cctally_db_sib._db_family_open_pids(
|
|
5181
|
+
path
|
|
5182
|
+
)
|
|
5183
|
+
if open_pids is None:
|
|
5184
|
+
raise OSError(
|
|
5185
|
+
"could not verify that the database family "
|
|
5186
|
+
"has no open handles"
|
|
5187
|
+
)
|
|
5188
|
+
if open_pids:
|
|
5189
|
+
raise OSError(
|
|
5190
|
+
"database family is still open in process(es) "
|
|
5191
|
+
+ ", ".join(
|
|
5192
|
+
str(pid) for pid in sorted(open_pids)
|
|
5193
|
+
)
|
|
5194
|
+
)
|
|
5195
|
+
_cctally_db_sib.quarantine_db_family(path, strict=True)
|
|
5196
|
+
except OSError as exc:
|
|
5197
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
5198
|
+
raise sqlite3.DatabaseError(
|
|
5199
|
+
f"cache.db pending quarantine could not resume: {exc}"
|
|
5200
|
+
) from exc
|
|
5201
|
+
removed, reclaim_reason = (
|
|
5202
|
+
_cctally_db_sib._remove_stale_repair_marker(path)
|
|
5203
|
+
)
|
|
5204
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
5205
|
+
if not removed:
|
|
5206
|
+
raise sqlite3.DatabaseError(
|
|
5207
|
+
f"cache.db maintenance is in progress: {reclaim_reason}"
|
|
5208
|
+
)
|
|
5209
|
+
continue
|
|
4837
5210
|
try:
|
|
4838
|
-
conn.
|
|
5211
|
+
conn = _cctally_store.open_index("cache")
|
|
5212
|
+
# Force a real header/schema read. SELECT 1 is constant-folded
|
|
5213
|
+
# and can report success without touching a malformed file.
|
|
5214
|
+
conn.execute("PRAGMA schema_version").fetchone()
|
|
5215
|
+
# The v1.80.2 incident left the schema and left edge readable
|
|
5216
|
+
# while the interior root's right-most child pointed past EOF.
|
|
5217
|
+
if conn.execute(
|
|
5218
|
+
"SELECT 1 FROM sqlite_schema "
|
|
5219
|
+
"WHERE type='table' AND name='session_entries'"
|
|
5220
|
+
).fetchone() is not None:
|
|
5221
|
+
conn.execute(
|
|
5222
|
+
"SELECT rowid FROM session_entries "
|
|
5223
|
+
"ORDER BY rowid DESC LIMIT 1"
|
|
5224
|
+
).fetchone()
|
|
5225
|
+
if marker.exists():
|
|
5226
|
+
conn.close()
|
|
5227
|
+
conn = None
|
|
5228
|
+
raise sqlite3.DatabaseError(
|
|
5229
|
+
f"cache.db maintenance is in progress ({marker})"
|
|
5230
|
+
)
|
|
5231
|
+
return conn
|
|
4839
5232
|
except Exception:
|
|
4840
|
-
|
|
4841
|
-
|
|
5233
|
+
if conn is not None:
|
|
5234
|
+
try:
|
|
5235
|
+
conn.close()
|
|
5236
|
+
except Exception:
|
|
5237
|
+
pass
|
|
5238
|
+
raise
|
|
5239
|
+
finally:
|
|
5240
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
5241
|
+
raise sqlite3.DatabaseError(
|
|
5242
|
+
"cache.db stale maintenance marker could not be reclaimed"
|
|
5243
|
+
)
|
|
4842
5244
|
finally:
|
|
4843
|
-
|
|
4844
|
-
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4845
|
-
finally:
|
|
4846
|
-
lock_fh.close()
|
|
5245
|
+
lock_fh.close()
|
|
4847
5246
|
|
|
4848
5247
|
|
|
4849
5248
|
def _recover_corrupt_cache(exc: sqlite3.DatabaseError) -> bool:
|
|
@@ -4858,15 +5257,16 @@ def _recover_corrupt_cache(exc: sqlite3.DatabaseError) -> bool:
|
|
|
4858
5257
|
|
|
4859
5258
|
path = pathlib.Path(_cctally_core.CACHE_DB_PATH)
|
|
4860
5259
|
try:
|
|
4861
|
-
|
|
5260
|
+
claim, reason = _cctally_db_sib._claim_repair_marker(path)
|
|
4862
5261
|
except OSError as marker_exc:
|
|
4863
5262
|
raise sqlite3.DatabaseError(
|
|
4864
5263
|
f"cache.db recovery could not claim maintenance: {marker_exc}"
|
|
4865
5264
|
) from exc
|
|
4866
|
-
if
|
|
5265
|
+
if claim is None:
|
|
4867
5266
|
raise sqlite3.DatabaseError(
|
|
4868
5267
|
f"cache.db maintenance is in progress: {reason}"
|
|
4869
5268
|
) from exc
|
|
5269
|
+
_cache_storm_test_pause("cache_repair_claimed")
|
|
4870
5270
|
|
|
4871
5271
|
lock_path = pathlib.Path(_cctally_core.CACHE_LOCK_MAINTENANCE_PATH)
|
|
4872
5272
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -4887,7 +5287,15 @@ def _recover_corrupt_cache(exc: sqlite3.DatabaseError) -> bool:
|
|
|
4887
5287
|
) from exc
|
|
4888
5288
|
|
|
4889
5289
|
_cctally_db_sib.write_corruption_forensics(path, db_label="cache")
|
|
4890
|
-
|
|
5290
|
+
_cache_storm_test_pause("cache_repair_forensics")
|
|
5291
|
+
try:
|
|
5292
|
+
incident = _cctally_db_sib.quarantine_db_family(path, strict=True)
|
|
5293
|
+
except OSError as quarantine_exc:
|
|
5294
|
+
raise sqlite3.DatabaseError(
|
|
5295
|
+
"cache.db recovery could not complete whole-family quarantine: "
|
|
5296
|
+
f"{quarantine_exc}"
|
|
5297
|
+
) from exc
|
|
5298
|
+
_cache_storm_test_pause("cache_repair_quarantined")
|
|
4891
5299
|
eprint(
|
|
4892
5300
|
f"[cache] corrupt cache DB ({exc}); quarantined its file family "
|
|
4893
5301
|
f"under {incident} and recreating from source JSONL"
|
|
@@ -4898,7 +5306,54 @@ def _recover_corrupt_cache(exc: sqlite3.DatabaseError) -> bool:
|
|
|
4898
5306
|
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
4899
5307
|
finally:
|
|
4900
5308
|
lock_fh.close()
|
|
4901
|
-
_cctally_db_sib._release_repair_marker(path)
|
|
5309
|
+
_cctally_db_sib._release_repair_marker(path, claim)
|
|
5310
|
+
|
|
5311
|
+
|
|
5312
|
+
def _run_cache_operation_with_recovery(
|
|
5313
|
+
conn: sqlite3.Connection,
|
|
5314
|
+
operation: Callable[[sqlite3.Connection], Any],
|
|
5315
|
+
) -> "tuple[Any, sqlite3.Connection]":
|
|
5316
|
+
results, replacement = _run_cache_plan_with_recovery(conn, (operation,))
|
|
5317
|
+
return results[0], replacement
|
|
5318
|
+
|
|
5319
|
+
|
|
5320
|
+
def _run_cache_plan_with_recovery(
|
|
5321
|
+
conn: sqlite3.Connection,
|
|
5322
|
+
operations: "tuple[Callable[[sqlite3.Connection], Any], ...]",
|
|
5323
|
+
) -> "tuple[tuple[Any, ...], sqlite3.Connection]":
|
|
5324
|
+
"""Run a provider plan, recovering once and restarting from its first leg.
|
|
5325
|
+
|
|
5326
|
+
The connection that observed corruption is closed before the destructive
|
|
5327
|
+
maintenance handshake. Because cache.db is one shared physical family, a
|
|
5328
|
+
recovery in a later provider leg invalidates every earlier result; the
|
|
5329
|
+
complete requested plan therefore restarts against the replacement family.
|
|
5330
|
+
A second classified failure closes the replacement and propagates without a
|
|
5331
|
+
second quarantine attempt.
|
|
5332
|
+
"""
|
|
5333
|
+
if not operations:
|
|
5334
|
+
return (), conn
|
|
5335
|
+
active = conn
|
|
5336
|
+
recovered = False
|
|
5337
|
+
while True:
|
|
5338
|
+
try:
|
|
5339
|
+
results = tuple(operation(active) for operation in operations)
|
|
5340
|
+
return results, active
|
|
5341
|
+
except sqlite3.DatabaseError as exc:
|
|
5342
|
+
if (
|
|
5343
|
+
recovered
|
|
5344
|
+
or not _cctally_db_sib._is_sqlite_corruption_error(exc)
|
|
5345
|
+
):
|
|
5346
|
+
active.close()
|
|
5347
|
+
raise
|
|
5348
|
+
active.close()
|
|
5349
|
+
if not _recover_corrupt_cache(exc):
|
|
5350
|
+
raise
|
|
5351
|
+
active = open_cache_db()
|
|
5352
|
+
_cache_storm_test_pause("cache_repair_recreated")
|
|
5353
|
+
recovered = True
|
|
5354
|
+
except BaseException:
|
|
5355
|
+
active.close()
|
|
5356
|
+
raise
|
|
4902
5357
|
|
|
4903
5358
|
|
|
4904
5359
|
def open_cache_db() -> sqlite3.Connection:
|
|
@@ -4919,6 +5374,7 @@ def open_cache_db() -> sqlite3.Connection:
|
|
|
4919
5374
|
os.chmod(_cctally_core.APP_DIR, 0o700)
|
|
4920
5375
|
except OSError as exc:
|
|
4921
5376
|
eprint(f"[cache] could not chmod data dir 0700 ({exc}); continuing")
|
|
5377
|
+
recovered = False
|
|
4922
5378
|
try:
|
|
4923
5379
|
conn = _cache_open_guarded()
|
|
4924
5380
|
except sqlite3.DatabaseError as exc:
|
|
@@ -4927,6 +5383,9 @@ def open_cache_db() -> sqlite3.Connection:
|
|
|
4927
5383
|
# One retry only. A second failure surfaces to the existing direct-JSONL
|
|
4928
5384
|
# fallback instead of looping through destructive recovery.
|
|
4929
5385
|
conn = _cache_open_guarded()
|
|
5386
|
+
recovered = True
|
|
5387
|
+
if recovered:
|
|
5388
|
+
_cache_storm_test_pause("cache_repair_recreated")
|
|
4930
5389
|
|
|
4931
5390
|
# Best-effort 0600 on cache.db itself (the 0700 dir above backstops the
|
|
4932
5391
|
# sidecars until the first write hardens them in sync_cache).
|
|
@@ -4935,59 +5394,78 @@ def open_cache_db() -> sqlite3.Connection:
|
|
|
4935
5394
|
except OSError as exc:
|
|
4936
5395
|
eprint(f"[cache] could not chmod cache.db 0600 ({exc}); continuing")
|
|
4937
5396
|
|
|
4938
|
-
|
|
4939
|
-
|
|
4940
|
-
|
|
4941
|
-
|
|
4942
|
-
|
|
4943
|
-
|
|
4944
|
-
# §6.2 version gate: run the full schema executescript + add_column_if_missing
|
|
4945
|
-
# probes + the codex last_total_tokens ALTER/purge ONLY when the stamped
|
|
4946
|
-
# user_version differs from the registry head. On a steady-state open of an
|
|
4947
|
-
# up-to-date cache.db this whole block is skipped — the open becomes
|
|
4948
|
-
# connect → PRAGMAs → one user_version read → the dispatcher fast-path.
|
|
4949
|
-
if not _cctally_store.schema_current(conn, "cache"):
|
|
4950
|
-
# Apply the shared cache.db schema (cctally-dev#93, D4): Claude tables +
|
|
4951
|
-
# indexes, the session_id / project_path column adds on session_files
|
|
4952
|
-
# (A2 `session` metadata, populated lazily in sync_cache() /
|
|
4953
|
-
# _ensure_session_files_row()), the Codex base tables + indexes, and the
|
|
4954
|
-
# cache_meta sentinel table. This is the single cache.db schema source —
|
|
4955
|
-
# the eager-apply path (_eagerly_apply_cache_migrations) uses the SAME
|
|
4956
|
-
# helper, so the two can no longer drift. The Codex last_total_tokens
|
|
4957
|
-
# ALTER + purge stays below (out of the shared helper — D4/P1#3).
|
|
4958
|
-
_cctally_db_sib._apply_cache_schema(conn)
|
|
4959
|
-
|
|
4960
|
-
# Migration: add last_total_tokens to codex_session_files. When the
|
|
4961
|
-
# column is newly added (i.e. this is the first run after upgrade),
|
|
4962
|
-
# purge the Codex cache so the duplicate-counted rows produced by the
|
|
4963
|
-
# previous iterator are reingested cleanly by sync_codex_cache(). The
|
|
4964
|
-
# cache is fully re-derivable from ~/.codex/sessions/*.jsonl so this is
|
|
4965
|
-
# safe. Under the version gate now (spec §6.2): on an up-to-date DB the
|
|
4966
|
-
# column already exists so add_column_if_missing was a no-op / no purge,
|
|
4967
|
-
# making the gate byte-equivalent.
|
|
4968
|
-
if add_column_if_missing(conn, "codex_session_files", "last_total_tokens", "INTEGER"):
|
|
4969
|
-
conn.execute("DELETE FROM codex_session_entries")
|
|
4970
|
-
conn.execute("DELETE FROM codex_session_files")
|
|
4971
|
-
conn.commit()
|
|
4972
|
-
eprint("[cache] migrated codex cache — re-ingesting")
|
|
5397
|
+
schema_current = _cctally_store.schema_current(conn, "cache")
|
|
5398
|
+
compatibility_current = conn.execute(
|
|
5399
|
+
"SELECT 1 FROM sqlite_master WHERE name='conversation_messages'"
|
|
5400
|
+
).fetchone() is not None
|
|
5401
|
+
journal_mode = str(conn.execute("PRAGMA journal_mode").fetchone()[0]).lower()
|
|
4973
5402
|
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
conn
|
|
4980
|
-
|
|
5403
|
+
if schema_current and compatibility_current and journal_mode == "wal":
|
|
5404
|
+
# Steady-state opens stay lock-free and apply connection-local policy
|
|
5405
|
+
# only. Persistent/schema PRAGMAs and every DDL/DML migration path are
|
|
5406
|
+
# reserved for the globally serialized branch below.
|
|
5407
|
+
_cctally_store.apply_connection_policy(conn, "cache")
|
|
5408
|
+
return conn
|
|
5409
|
+
|
|
5410
|
+
from _lib_cache_writer_lock import (
|
|
5411
|
+
acquire_ordered_flocks,
|
|
5412
|
+
release_cache_writer_flocks,
|
|
4981
5413
|
)
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
|
|
4987
|
-
|
|
4988
|
-
|
|
4989
|
-
)
|
|
4990
|
-
|
|
5414
|
+
|
|
5415
|
+
held = acquire_ordered_flocks(
|
|
5416
|
+
[
|
|
5417
|
+
(_cctally_core.CACHE_LOCK_MAINTENANCE_PATH, fcntl.LOCK_EX),
|
|
5418
|
+
(_cctally_core.CACHE_LOCK_PATH, fcntl.LOCK_EX),
|
|
5419
|
+
],
|
|
5420
|
+
timeout=15.0,
|
|
5421
|
+
)
|
|
5422
|
+
if held is None:
|
|
5423
|
+
conn.close()
|
|
5424
|
+
raise sqlite3.DatabaseError(
|
|
5425
|
+
"cache.db schema/policy update deferred: cache writer is busy"
|
|
5426
|
+
)
|
|
5427
|
+
try:
|
|
5428
|
+
# Another first-open/upgrade process may have completed while this one
|
|
5429
|
+
# waited. Re-read every mutation gate under the global writer flock.
|
|
5430
|
+
schema_current = _cctally_store.schema_current(conn, "cache")
|
|
5431
|
+
compatibility_current = conn.execute(
|
|
5432
|
+
"SELECT 1 FROM sqlite_master WHERE name='conversation_messages'"
|
|
5433
|
+
).fetchone() is not None
|
|
5434
|
+
journal_mode = str(
|
|
5435
|
+
conn.execute("PRAGMA journal_mode").fetchone()[0]
|
|
5436
|
+
).lower()
|
|
5437
|
+
|
|
5438
|
+
if not schema_current or journal_mode != "wal":
|
|
5439
|
+
_cctally_store.apply_policy(conn, "cache")
|
|
5440
|
+
else:
|
|
5441
|
+
_cctally_store.apply_connection_policy(conn, "cache")
|
|
5442
|
+
|
|
5443
|
+
# §6.2 version gate: schema apply, ALTER/purge, and the complete cache
|
|
5444
|
+
# migration dispatcher all run under maintenance-exclusive → global writer
|
|
5445
|
+
# exclusion. No first-open or upgrade write can overlap provider sync.
|
|
5446
|
+
if not schema_current:
|
|
5447
|
+
_cctally_db_sib._apply_cache_schema(conn)
|
|
5448
|
+
if add_column_if_missing(
|
|
5449
|
+
conn,
|
|
5450
|
+
"codex_session_files",
|
|
5451
|
+
"last_total_tokens",
|
|
5452
|
+
"INTEGER",
|
|
5453
|
+
):
|
|
5454
|
+
conn.execute("DELETE FROM codex_session_entries")
|
|
5455
|
+
conn.execute("DELETE FROM codex_session_files")
|
|
5456
|
+
conn.commit()
|
|
5457
|
+
eprint("[cache] migrated codex cache — re-ingesting")
|
|
5458
|
+
_cctally_db_sib._run_pending_cache_migrations_under_writer_lock(conn)
|
|
5459
|
+
|
|
5460
|
+
# Migration 028 removes the legacy transcript objects after arming the
|
|
5461
|
+
# independent rebuild. Recreate EMPTY compatibility objects only so
|
|
5462
|
+
# older migration/fixture probes remain valid.
|
|
5463
|
+
if conn.execute(
|
|
5464
|
+
"SELECT 1 FROM sqlite_master WHERE name='conversation_messages'"
|
|
5465
|
+
).fetchone() is None:
|
|
5466
|
+
_cctally_db_sib._apply_cache_schema(conn)
|
|
5467
|
+
finally:
|
|
5468
|
+
release_cache_writer_flocks(held)
|
|
4991
5469
|
return conn
|
|
4992
5470
|
|
|
4993
5471
|
|
|
@@ -5892,6 +6370,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5892
6370
|
"(Codex orphans are pruned automatically during codex sync); "
|
|
5893
6371
|
"nothing to do for --source codex."
|
|
5894
6372
|
)
|
|
6373
|
+
conn.close()
|
|
5895
6374
|
return 0
|
|
5896
6375
|
res = _prune_orphaned_cache_entries(
|
|
5897
6376
|
conn, lock_timeout=_REBUILD_LOCK_TIMEOUT_SECONDS
|
|
@@ -5901,6 +6380,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5901
6380
|
"[cache-sync] prune-orphans skipped: "
|
|
5902
6381
|
"another process holds the lock"
|
|
5903
6382
|
)
|
|
6383
|
+
conn.close()
|
|
5904
6384
|
return 1
|
|
5905
6385
|
eprint(
|
|
5906
6386
|
f"[cache-sync] pruned {res.pruned_files} orphaned file(s), "
|
|
@@ -5912,6 +6392,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5912
6392
|
f"(shared session or missing conversation evidence); "
|
|
5913
6393
|
f"run `cache-sync --rebuild` to clear them"
|
|
5914
6394
|
)
|
|
6395
|
+
conn.close()
|
|
5915
6396
|
return 0
|
|
5916
6397
|
|
|
5917
6398
|
# --prune-conversations: on-demand, UNTHROTTLED transcript retention prune
|
|
@@ -5926,6 +6407,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5926
6407
|
"[cache-sync] transcript retention is disabled "
|
|
5927
6408
|
"(conversation.retention_days=0); nothing pruned."
|
|
5928
6409
|
)
|
|
6410
|
+
conn.close()
|
|
5929
6411
|
return 0
|
|
5930
6412
|
conv_conn = open_conversations_db(attach_cache=False)
|
|
5931
6413
|
try:
|
|
@@ -5942,6 +6424,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5942
6424
|
"[cache-sync] prune-conversations skipped: another process "
|
|
5943
6425
|
"holds the maintenance or a provider lock; retry shortly."
|
|
5944
6426
|
)
|
|
6427
|
+
conn.close()
|
|
5945
6428
|
return 1
|
|
5946
6429
|
eprint(
|
|
5947
6430
|
f"[cache-sync] pruned transcripts older than {retention_days}d: "
|
|
@@ -5951,6 +6434,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5951
6434
|
f"{result.codex_events} event(s). "
|
|
5952
6435
|
f"Run `cctally db vacuum --db conversations` to reclaim the freed space."
|
|
5953
6436
|
)
|
|
6437
|
+
conn.close()
|
|
5954
6438
|
return 0
|
|
5955
6439
|
|
|
5956
6440
|
# Note: when --rebuild is set we delegate the DELETE to sync_cache /
|
|
@@ -5971,11 +6455,47 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5971
6455
|
_p_root = _perf.phase("cache-sync")
|
|
5972
6456
|
_p_root.__enter__()
|
|
5973
6457
|
|
|
6458
|
+
plan: list[Callable[[sqlite3.Connection], Any]] = []
|
|
6459
|
+
|
|
5974
6460
|
if source in ("claude", "all"):
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
6461
|
+
def _sync_claude_leg(active_conn: sqlite3.Connection) -> IngestStats:
|
|
6462
|
+
with _perf.phase("sync_cache"):
|
|
6463
|
+
return sync_cache(
|
|
6464
|
+
active_conn,
|
|
6465
|
+
progress=_progress_stderr,
|
|
6466
|
+
rebuild=args.rebuild,
|
|
6467
|
+
lock_timeout=lt,
|
|
6468
|
+
)
|
|
6469
|
+
|
|
6470
|
+
plan.append(_sync_claude_leg)
|
|
6471
|
+
|
|
6472
|
+
if source in ("codex", "all"):
|
|
6473
|
+
def _sync_codex_leg(
|
|
6474
|
+
active_conn: sqlite3.Connection,
|
|
6475
|
+
) -> CodexIngestStats:
|
|
6476
|
+
with _perf.phase("sync_codex_cache"):
|
|
6477
|
+
return sync_codex_cache(
|
|
6478
|
+
active_conn,
|
|
6479
|
+
progress=_progress_codex_stderr,
|
|
6480
|
+
rebuild=args.rebuild,
|
|
6481
|
+
lock_timeout=lt,
|
|
6482
|
+
)
|
|
6483
|
+
|
|
6484
|
+
plan.append(_sync_codex_leg)
|
|
6485
|
+
|
|
6486
|
+
try:
|
|
6487
|
+
plan_results, conn = _run_cache_plan_with_recovery(conn, tuple(plan))
|
|
6488
|
+
except (OSError, sqlite3.DatabaseError) as exc:
|
|
6489
|
+
eprint(f"[cache-sync] failed: {exc}")
|
|
6490
|
+
_p_root.__exit__(type(exc), exc, exc.__traceback__)
|
|
6491
|
+
if _perf.enabled():
|
|
6492
|
+
_perf.flush_stderr(_perf.current_root())
|
|
6493
|
+
return 1
|
|
6494
|
+
result_index = 0
|
|
6495
|
+
|
|
6496
|
+
if source in ("claude", "all"):
|
|
6497
|
+
stats = plan_results[result_index]
|
|
6498
|
+
result_index += 1
|
|
5979
6499
|
_progress_stderr(stats, force=True)
|
|
5980
6500
|
if stats.lock_contended and args.rebuild:
|
|
5981
6501
|
eprint(
|
|
@@ -5983,6 +6503,12 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5983
6503
|
"another process holds the lock"
|
|
5984
6504
|
)
|
|
5985
6505
|
contended = True
|
|
6506
|
+
elif stats.files_failed and args.rebuild:
|
|
6507
|
+
eprint(
|
|
6508
|
+
"[cache-sync] rebuild incomplete (claude): "
|
|
6509
|
+
f"{stats.files_failed} file(s) failed"
|
|
6510
|
+
)
|
|
6511
|
+
contended = True
|
|
5986
6512
|
elif not stats.lock_contended:
|
|
5987
6513
|
eprint(
|
|
5988
6514
|
f"[cache-sync] claude done: {stats.files_processed} processed, "
|
|
@@ -5994,11 +6520,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
5994
6520
|
)
|
|
5995
6521
|
|
|
5996
6522
|
if source in ("codex", "all"):
|
|
5997
|
-
|
|
5998
|
-
stats = sync_codex_cache(
|
|
5999
|
-
conn, progress=_progress_codex_stderr, rebuild=args.rebuild,
|
|
6000
|
-
lock_timeout=lt,
|
|
6001
|
-
)
|
|
6523
|
+
stats = plan_results[result_index]
|
|
6002
6524
|
_progress_codex_stderr(stats, force=True)
|
|
6003
6525
|
if stats.lock_contended and args.rebuild:
|
|
6004
6526
|
eprint(
|
|
@@ -6006,6 +6528,12 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
6006
6528
|
"another process holds the lock"
|
|
6007
6529
|
)
|
|
6008
6530
|
contended = True
|
|
6531
|
+
elif stats.files_failed and args.rebuild:
|
|
6532
|
+
eprint(
|
|
6533
|
+
"[cache-sync] rebuild incomplete (codex): "
|
|
6534
|
+
f"{stats.files_failed} file(s) failed"
|
|
6535
|
+
)
|
|
6536
|
+
contended = True
|
|
6009
6537
|
elif not stats.lock_contended:
|
|
6010
6538
|
eprint(
|
|
6011
6539
|
f"[cache-sync] codex done: {stats.files_processed} processed, "
|
|
@@ -6016,6 +6544,8 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
6016
6544
|
f"{stats.token_events_skipped} drift-skipped"
|
|
6017
6545
|
)
|
|
6018
6546
|
|
|
6547
|
+
conn.close()
|
|
6548
|
+
|
|
6019
6549
|
# #320: transcript/search ingestion is a second physical database with its
|
|
6020
6550
|
# own cursors and flocks. Run it only after the core providers have
|
|
6021
6551
|
# committed so a slow/failed transcript pass can never roll back accounting
|