cctally 1.98.0 → 1.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +4 -4
- package/bin/_cctally_account.py +925 -0
- package/bin/_cctally_cache.py +829 -31
- package/bin/_cctally_core.py +52 -0
- package/bin/_cctally_dashboard.py +375 -19
- package/bin/_cctally_dashboard_share.py +50 -1
- package/bin/_cctally_dashboard_sources.py +1501 -118
- package/bin/_cctally_db.py +810 -34
- package/bin/_cctally_doctor.py +184 -1
- package/bin/_cctally_journal.py +732 -76
- package/bin/_cctally_parser.py +65 -0
- package/bin/_cctally_quota.py +896 -17
- package/bin/_cctally_rederive.py +157 -5
- package/bin/_cctally_source_analytics.py +60 -6
- package/bin/_cctally_tui.py +508 -47
- package/bin/_lib_aggregators.py +117 -11
- package/bin/_lib_budget.py +60 -0
- package/bin/_lib_codex_window_attribution.py +259 -0
- package/bin/_lib_dashboard_sources.py +26 -1
- package/bin/_lib_doctor.py +71 -0
- package/bin/_lib_journal.py +212 -0
- package/bin/_lib_jsonl.py +6 -0
- package/bin/_lib_rederive.py +8 -0
- package/bin/_lib_snapshot_cache.py +236 -7
- package/bin/_lib_source_analytics.py +20 -2
- package/bin/cctally +12 -0
- package/dashboard/static/assets/index-Bcbm-DNP.js +97 -0
- package/dashboard/static/assets/index-hJP4wlIO.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-CC8TTZUC.css +0 -1
- package/dashboard/static/assets/index-CChXFhs_.js +0 -97
package/bin/_lib_journal.py
CHANGED
|
@@ -1308,6 +1308,218 @@ def make_codex_file_account(
|
|
|
1308
1308
|
return make_op(at=at, src="codex-file-account", payload=payload)
|
|
1309
1309
|
|
|
1310
1310
|
|
|
1311
|
+
#: Only account-level weekly Codex quota is attributable (#500 spec §5). A 5h
|
|
1312
|
+
#: window and a model-scoped pool are out of scope, and the builder refuses
|
|
1313
|
+
#: rather than recording an assertion the overlay would always suppress.
|
|
1314
|
+
ACCOUNT_WEEKLY_WINDOW_MINUTES = 10_080
|
|
1315
|
+
|
|
1316
|
+
#: The sentinel is never an assertable subject and is never written into a
|
|
1317
|
+
#: payload (the two-shaped stamp rule, docs/accounts-gotchas.md).
|
|
1318
|
+
_ATTRIBUTION_SENTINEL = "unattributed"
|
|
1319
|
+
|
|
1320
|
+
WINDOW_ATTRIBUTION_KIND = "codex_window_attribution"
|
|
1321
|
+
WINDOW_ATTRIBUTION_RETRACT_KIND = "codex_window_attribution_retract"
|
|
1322
|
+
WINDOW_ATTRIBUTION_SRC = "account-attribute"
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _sorted_unique_strings(values, what: str) -> list:
|
|
1326
|
+
"""Sorted, de-duplicated, non-empty strings, or ``ValueError``.
|
|
1327
|
+
|
|
1328
|
+
A bare string is refused rather than iterated: ``"abc"`` would otherwise
|
|
1329
|
+
silently become three one-character witnesses. Types are checked BEFORE the
|
|
1330
|
+
sort, so a non-string member raises ``ValueError`` rather than the
|
|
1331
|
+
``TypeError`` a mixed-type ``sorted`` would raise.
|
|
1332
|
+
"""
|
|
1333
|
+
if isinstance(values, (str, bytes)):
|
|
1334
|
+
raise ValueError(f"{what} must be a sequence of strings, not a string")
|
|
1335
|
+
try:
|
|
1336
|
+
items = list(values)
|
|
1337
|
+
except TypeError as exc:
|
|
1338
|
+
raise ValueError(f"{what} must be a sequence of strings") from exc
|
|
1339
|
+
for item in items:
|
|
1340
|
+
if not isinstance(item, str) or not item:
|
|
1341
|
+
raise ValueError(f"every {what} entry must be a non-empty string")
|
|
1342
|
+
return sorted(set(items))
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def _normalize_attribution_instant(at: str) -> str:
|
|
1346
|
+
"""One ISO spelling for a #500 assertion's `at` (review finding F9).
|
|
1347
|
+
|
|
1348
|
+
`make_op` stores the caller's string verbatim, and the active read orders
|
|
1349
|
+
assertions with `ORDER BY asserted_at_utc ASC` — a LEXICOGRAPHIC sort over
|
|
1350
|
+
that stored text. Lexicographic order is chronological order only while
|
|
1351
|
+
every value uses one spelling: `2026-08-14T12:00:00+00:00` sorts before
|
|
1352
|
+
`2026-08-14T06:00:00Z` because `+` (0x2B) precedes `Z` (0x5A), so a single
|
|
1353
|
+
offset-form assertion would silently take precedence over every later
|
|
1354
|
+
Z-form one. Normalizing HERE, at the one chokepoint both builders pass
|
|
1355
|
+
through, is what makes the cheap SQL ordering correct.
|
|
1356
|
+
|
|
1357
|
+
MICROSECOND precision with a `Z` suffix, and the six fractional digits are
|
|
1358
|
+
written ALWAYS, even when they are zero (review round 2, finding R2-1).
|
|
1359
|
+
Two separate rules meet here and both require the fixed width.
|
|
1360
|
+
|
|
1361
|
+
Truncating to seconds is a correctness defect, not a cosmetic one. The
|
|
1362
|
+
normalized value is what `content_id` digests, so two assertions inside one
|
|
1363
|
+
wall-clock second with otherwise-identical payloads collapse to the SAME
|
|
1364
|
+
`op_id`. Spec §7.2's assert -> retract -> re-assert then loses its last
|
|
1365
|
+
step: the re-assertion carries the id the tombstone already names, so
|
|
1366
|
+
`INSERT OR IGNORE` drops it against the retracted row and the operator's
|
|
1367
|
+
second assertion vanishes with no error. §7.2 requires a re-assertion to
|
|
1368
|
+
carry an id no earlier tombstone names, and sub-second precision is what
|
|
1369
|
+
supplies it.
|
|
1370
|
+
|
|
1371
|
+
Emitting the fraction only when non-zero would break the ordering rule
|
|
1372
|
+
above: `.` (0x2E) precedes `Z` (0x5A), so `2026-08-14T00:00:05.500000Z`
|
|
1373
|
+
would sort BEFORE `2026-08-14T00:00:05Z` — a fractional value ahead of the
|
|
1374
|
+
whole second it follows. One fixed-width spelling is what keeps
|
|
1375
|
+
lexicographic order equal to chronological order.
|
|
1376
|
+
|
|
1377
|
+
A naive value is refused rather than assumed to be UTC: the assertion is
|
|
1378
|
+
durable, and guessing a zone for it is not recoverable.
|
|
1379
|
+
"""
|
|
1380
|
+
if not isinstance(at, str) or not at:
|
|
1381
|
+
raise ValueError("at must be a non-empty ISO instant")
|
|
1382
|
+
try:
|
|
1383
|
+
parsed = dt.datetime.fromisoformat(at.replace("Z", "+00:00"))
|
|
1384
|
+
except ValueError as exc:
|
|
1385
|
+
raise ValueError(f"at is not an ISO instant: {at!r}") from exc
|
|
1386
|
+
if parsed.tzinfo is None:
|
|
1387
|
+
raise ValueError(f"at must carry a timezone offset: {at!r}")
|
|
1388
|
+
return parsed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _window_attribution_payload(
|
|
1392
|
+
kind: str,
|
|
1393
|
+
*,
|
|
1394
|
+
account_key: str,
|
|
1395
|
+
source_root_key: str,
|
|
1396
|
+
logical_limit_key: str,
|
|
1397
|
+
observed_slot: str,
|
|
1398
|
+
window_minutes: int,
|
|
1399
|
+
raw_resets_at_utc,
|
|
1400
|
+
canonical_resets_at_utc: str,
|
|
1401
|
+
) -> dict:
|
|
1402
|
+
"""Shared payload for the two #500 operator-attribution op kinds.
|
|
1403
|
+
|
|
1404
|
+
The binding is the four normalized axes plus the tolerance-connected
|
|
1405
|
+
component WITNESSED by ``raw_resets_at_utc``. The canonical anchor is
|
|
1406
|
+
carried for audit and preview only and is NEVER matched on: it is
|
|
1407
|
+
population-dependent, because a later bridging observation can union two
|
|
1408
|
+
components and retire the earlier anchor (``_lib_quota`` component union),
|
|
1409
|
+
which is exactly why the canonical value is not journaled anywhere else.
|
|
1410
|
+
|
|
1411
|
+
The ``account_key`` here is the SUBJECT of the assertion — the account the
|
|
1412
|
+
operator says the window belongs to — not the two-shaped stamp naming which
|
|
1413
|
+
account wrote the record. Both kinds are registered in
|
|
1414
|
+
``_ACCOUNTS_MACHINERY_KINDS``, but NOT because the legacy normalizer would
|
|
1415
|
+
otherwise mistake this field for a missing stamp (review finding F7) — it
|
|
1416
|
+
would not, being gated on ``_REAL_ACCOUNT_EVT_OP_KINDS``, which these kinds
|
|
1417
|
+
were never in. Registration is what puts the kinds into the re-derive
|
|
1418
|
+
planner's ``op_kinds`` set, which makes their
|
|
1419
|
+
``_lib_rederive._OP_CLASSIFICATIONS`` entries mandatory.
|
|
1420
|
+
"""
|
|
1421
|
+
if not isinstance(account_key, str) or not account_key:
|
|
1422
|
+
raise ValueError("account_key must be a non-empty string")
|
|
1423
|
+
if account_key == _ATTRIBUTION_SENTINEL:
|
|
1424
|
+
raise ValueError("the unattributed sentinel is not an assertable subject")
|
|
1425
|
+
for name, value in (("source_root_key", source_root_key),
|
|
1426
|
+
("logical_limit_key", logical_limit_key),
|
|
1427
|
+
("observed_slot", observed_slot),
|
|
1428
|
+
("canonical_resets_at_utc", canonical_resets_at_utc)):
|
|
1429
|
+
if not isinstance(value, str) or not value:
|
|
1430
|
+
raise ValueError(f"{name} must be a non-empty string")
|
|
1431
|
+
if (not isinstance(window_minutes, int)
|
|
1432
|
+
or isinstance(window_minutes, bool)
|
|
1433
|
+
or window_minutes != ACCOUNT_WEEKLY_WINDOW_MINUTES):
|
|
1434
|
+
raise ValueError(
|
|
1435
|
+
f"window_minutes must be {ACCOUNT_WEEKLY_WINDOW_MINUTES} "
|
|
1436
|
+
"(account weekly quota)")
|
|
1437
|
+
witnesses = _sorted_unique_strings(raw_resets_at_utc, "raw reset witness")
|
|
1438
|
+
if not witnesses:
|
|
1439
|
+
raise ValueError("raw_resets_at_utc must carry at least one witness")
|
|
1440
|
+
return {
|
|
1441
|
+
"kind": kind,
|
|
1442
|
+
"source": "codex",
|
|
1443
|
+
"account_key": account_key,
|
|
1444
|
+
"source_root_key": source_root_key,
|
|
1445
|
+
"logical_limit_key": logical_limit_key,
|
|
1446
|
+
"observed_slot": observed_slot,
|
|
1447
|
+
"window_minutes": window_minutes,
|
|
1448
|
+
"raw_resets_at_utc": witnesses,
|
|
1449
|
+
"canonical_resets_at_utc": canonical_resets_at_utc,
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def make_codex_window_attribution(
|
|
1454
|
+
at: str,
|
|
1455
|
+
*,
|
|
1456
|
+
account_key: str,
|
|
1457
|
+
source_root_key: str,
|
|
1458
|
+
logical_limit_key: str,
|
|
1459
|
+
observed_slot: str,
|
|
1460
|
+
window_minutes: int,
|
|
1461
|
+
raw_resets_at_utc,
|
|
1462
|
+
canonical_resets_at_utc: str,
|
|
1463
|
+
) -> dict:
|
|
1464
|
+
"""Build a ``codex_window_attribution`` op — the operator's durable
|
|
1465
|
+
assertion that one physical Codex quota window group belongs to one account
|
|
1466
|
+
(#500 spec §5).
|
|
1467
|
+
|
|
1468
|
+
Ordinary ``content_id``, deliberately NOT a stable singleton id like the
|
|
1469
|
+
accounts cutover's: a fixed id would make assert -> retract -> reassert
|
|
1470
|
+
impossible, because the second assertion would collide with the first.
|
|
1471
|
+
"""
|
|
1472
|
+
payload = _window_attribution_payload(
|
|
1473
|
+
WINDOW_ATTRIBUTION_KIND,
|
|
1474
|
+
account_key=account_key,
|
|
1475
|
+
source_root_key=source_root_key,
|
|
1476
|
+
logical_limit_key=logical_limit_key,
|
|
1477
|
+
observed_slot=observed_slot,
|
|
1478
|
+
window_minutes=window_minutes,
|
|
1479
|
+
raw_resets_at_utc=raw_resets_at_utc,
|
|
1480
|
+
canonical_resets_at_utc=canonical_resets_at_utc,
|
|
1481
|
+
)
|
|
1482
|
+
return make_op(at=_normalize_attribution_instant(at),
|
|
1483
|
+
src=WINDOW_ATTRIBUTION_SRC, payload=payload)
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def make_codex_window_attribution_retract(
|
|
1487
|
+
at: str,
|
|
1488
|
+
*,
|
|
1489
|
+
account_key: str,
|
|
1490
|
+
source_root_key: str,
|
|
1491
|
+
logical_limit_key: str,
|
|
1492
|
+
observed_slot: str,
|
|
1493
|
+
window_minutes: int,
|
|
1494
|
+
raw_resets_at_utc,
|
|
1495
|
+
canonical_resets_at_utc: str,
|
|
1496
|
+
retracted_assertion_ids,
|
|
1497
|
+
) -> dict:
|
|
1498
|
+
"""Build a ``codex_window_attribution_retract`` op (#500 spec §5, §7.2).
|
|
1499
|
+
|
|
1500
|
+
Targets specific assertion op IDs rather than the group, so an older
|
|
1501
|
+
tombstone can never suppress a later reassertion: the reassertion carries a
|
|
1502
|
+
new content id that no earlier tombstone names.
|
|
1503
|
+
"""
|
|
1504
|
+
payload = _window_attribution_payload(
|
|
1505
|
+
WINDOW_ATTRIBUTION_RETRACT_KIND,
|
|
1506
|
+
account_key=account_key,
|
|
1507
|
+
source_root_key=source_root_key,
|
|
1508
|
+
logical_limit_key=logical_limit_key,
|
|
1509
|
+
observed_slot=observed_slot,
|
|
1510
|
+
window_minutes=window_minutes,
|
|
1511
|
+
raw_resets_at_utc=raw_resets_at_utc,
|
|
1512
|
+
canonical_resets_at_utc=canonical_resets_at_utc,
|
|
1513
|
+
)
|
|
1514
|
+
targets = _sorted_unique_strings(
|
|
1515
|
+
retracted_assertion_ids, "retracted assertion id")
|
|
1516
|
+
if not targets:
|
|
1517
|
+
raise ValueError("retracted_assertion_ids must name at least one assertion")
|
|
1518
|
+
payload["retracted_assertion_ids"] = targets
|
|
1519
|
+
return make_op(at=_normalize_attribution_instant(at),
|
|
1520
|
+
src=WINDOW_ATTRIBUTION_SRC, payload=payload)
|
|
1521
|
+
|
|
1522
|
+
|
|
1311
1523
|
# --------------------------------------------------------------------------
|
|
1312
1524
|
# segment naming + canonical order
|
|
1313
1525
|
# --------------------------------------------------------------------------
|
package/bin/_lib_jsonl.py
CHANGED
|
@@ -81,6 +81,12 @@ class CodexEntry:
|
|
|
81
81
|
reasoning_output_tokens: int
|
|
82
82
|
total_tokens: int
|
|
83
83
|
source_path: str
|
|
84
|
+
# Cache-backed readers already compute the canonical price while loading.
|
|
85
|
+
# Direct JSONL/CLI readers leave this unset and retain historical pricing.
|
|
86
|
+
cost_usd: float | None = None
|
|
87
|
+
cache_entry_id: int = field(default=0, compare=False, repr=False)
|
|
88
|
+
source_root_key: str = field(default="", compare=False, repr=False)
|
|
89
|
+
conversation_key: str = field(default="", compare=False, repr=False)
|
|
84
90
|
|
|
85
91
|
|
|
86
92
|
def _entry_token_total(entry: "UsageEntry") -> int:
|
package/bin/_lib_rederive.py
CHANGED
|
@@ -105,6 +105,14 @@ _OP_CLASSIFICATIONS = {
|
|
|
105
105
|
# leg rather than into the claude-usage scratch index.
|
|
106
106
|
"codex_file_account": KindClassification(
|
|
107
107
|
"retained", "Codex file attribution decision is outside claude-usage"),
|
|
108
|
+
# #500: operator attribution of recorded Codex quota windows. `retained`,
|
|
109
|
+
# exactly like the `codex_file_account` precedent above — provider state
|
|
110
|
+
# OUTSIDE the claude-usage family, replayed by the Codex cache leg rather
|
|
111
|
+
# than into the claude-usage scratch index.
|
|
112
|
+
"codex_window_attribution": KindClassification(
|
|
113
|
+
"retained", "operator Codex window attribution is outside claude-usage"),
|
|
114
|
+
"codex_window_attribution_retract": KindClassification(
|
|
115
|
+
"retained", "operator Codex window retraction is outside claude-usage"),
|
|
108
116
|
}
|
|
109
117
|
|
|
110
118
|
|
|
@@ -91,6 +91,11 @@ class SnapshotSignature(NamedTuple):
|
|
|
91
91
|
# sequence supplies that missing identity leg; the stats digest arrives
|
|
92
92
|
# from the independently-committed quota/budget projection database.
|
|
93
93
|
codex_physical_mutation_seq: int = 0
|
|
94
|
+
# #582: path-ledger accounting changes must leave the snapshot idle path,
|
|
95
|
+
# even when MAX(id) and the broader physical counter stay flat. This leg
|
|
96
|
+
# is internal dispatch identity only; published data-version bytes retain
|
|
97
|
+
# their existing contract.
|
|
98
|
+
codex_accounting_mutation_seq: int = 0
|
|
94
99
|
codex_stats_digest: str = ""
|
|
95
100
|
# #341 finding 9: a digest of the account registry + the providers' on-disk
|
|
96
101
|
# identity-file/active-account state. Empty for every <=1-account install (no
|
|
@@ -251,6 +256,9 @@ def compute_signature(
|
|
|
251
256
|
generation=int(generation),
|
|
252
257
|
entry_mutation_seq=_entry_mutation_seq(cache_conn),
|
|
253
258
|
codex_physical_mutation_seq=_codex_physical_mutation_seq(cache_conn),
|
|
259
|
+
codex_accounting_mutation_seq=(
|
|
260
|
+
_codex_accounting_mutation_seq(cache_conn) or 0
|
|
261
|
+
),
|
|
254
262
|
codex_stats_digest=str(codex_stats_digest),
|
|
255
263
|
accounts_digest=str(accounts_digest),
|
|
256
264
|
codex_ingest_backlog_sig=_codex_ingest_backlog_sig(cache_conn),
|
|
@@ -960,6 +968,228 @@ def reset_session_cache_state() -> None:
|
|
|
960
968
|
_SESSION_LAST_SEEN.clear()
|
|
961
969
|
|
|
962
970
|
|
|
971
|
+
# === #582 — persistent Codex accounting rows by dirty physical path =========
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
@dataclass(frozen=True)
|
|
975
|
+
class CodexAccountingCacheResult:
|
|
976
|
+
"""One cold or incrementally refreshed accounting population."""
|
|
977
|
+
|
|
978
|
+
entries: tuple[object, ...]
|
|
979
|
+
dirty_paths: tuple[tuple[str, str], ...]
|
|
980
|
+
dirty_accounts: tuple[str, ...]
|
|
981
|
+
cold: bool
|
|
982
|
+
changed_old: tuple[object, ...] = ()
|
|
983
|
+
changed_new: tuple[object, ...] = ()
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
_CODEX_ACCOUNTING_CACHE_STATE: dict[str, object] = {}
|
|
987
|
+
_CODEX_ACCOUNTING_MAX_DIRTY_PATHS = 300
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def reset_codex_accounting_cache_state() -> None:
|
|
991
|
+
"""Drop #582's value-only Codex accounting cache and ledger cursor."""
|
|
992
|
+
_assert_owner()
|
|
993
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
def checkpoint_codex_accounting_cache_state() -> dict[str, object]:
|
|
997
|
+
"""Copy the value-only state so a failed source build can roll back."""
|
|
998
|
+
_assert_owner()
|
|
999
|
+
return dict(_CODEX_ACCOUNTING_CACHE_STATE)
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def restore_codex_accounting_cache_state(state: dict[str, object]) -> None:
|
|
1003
|
+
"""Restore a checkpoint after downstream source construction fails."""
|
|
1004
|
+
_assert_owner()
|
|
1005
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
1006
|
+
_CODEX_ACCOUNTING_CACHE_STATE.update(state)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def _codex_accounting_mutation_seq(conn: sqlite3.Connection) -> int | None:
|
|
1010
|
+
"""Return the dedicated ledger sequence, or ``None`` when unsupported."""
|
|
1011
|
+
try:
|
|
1012
|
+
row = conn.execute(
|
|
1013
|
+
"SELECT value FROM cache_meta "
|
|
1014
|
+
"WHERE key='codex_accounting_mutation_seq'"
|
|
1015
|
+
).fetchone()
|
|
1016
|
+
table = conn.execute(
|
|
1017
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' "
|
|
1018
|
+
"AND name='codex_accounting_change_log'"
|
|
1019
|
+
).fetchone()
|
|
1020
|
+
except sqlite3.Error:
|
|
1021
|
+
return None
|
|
1022
|
+
if table is None or row is None or row[0] is None:
|
|
1023
|
+
return None
|
|
1024
|
+
try:
|
|
1025
|
+
return int(row[0])
|
|
1026
|
+
except (TypeError, ValueError):
|
|
1027
|
+
return None
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def codex_accounting_cache_pending(conn: sqlite3.Connection) -> bool:
|
|
1031
|
+
"""Whether the process cache has not consumed the durable ledger head."""
|
|
1032
|
+
_assert_owner()
|
|
1033
|
+
current = _codex_accounting_mutation_seq(conn)
|
|
1034
|
+
if current is None or not _CODEX_ACCOUNTING_CACHE_STATE:
|
|
1035
|
+
return True
|
|
1036
|
+
return current != int(_CODEX_ACCOUNTING_CACHE_STATE.get("seq", -1))
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def build_cached_codex_accounting(
|
|
1040
|
+
*,
|
|
1041
|
+
cache_conn: sqlite3.Connection,
|
|
1042
|
+
range_start: dt.datetime,
|
|
1043
|
+
range_end: dt.datetime,
|
|
1044
|
+
extra_signature: object,
|
|
1045
|
+
load_all: "Callable[[], tuple[object, ...]]",
|
|
1046
|
+
load_paths: "Callable[[tuple[tuple[str, str], ...]], tuple[object, ...]]",
|
|
1047
|
+
path_of: "Callable[[object], tuple[str, str]]",
|
|
1048
|
+
account_of: "Callable[[object], str]",
|
|
1049
|
+
order_key: "Callable[[object], object]",
|
|
1050
|
+
identity_of: "Callable[[object], object] | None" = None,
|
|
1051
|
+
finalize_entries: "Callable[[tuple[object, ...], tuple[object, ...] | None], tuple[object, ...]] | None" = None,
|
|
1052
|
+
) -> CodexAccountingCacheResult:
|
|
1053
|
+
"""Cold-load once, then replace only ledger-dirty Codex file populations.
|
|
1054
|
+
|
|
1055
|
+
The cache retains plain immutable values and one integer cursor. A missing
|
|
1056
|
+
ledger, cursor gap, full marker, semantic-key change, range-start change,
|
|
1057
|
+
or clock regression goes cold. When the upper bound advances, paths with
|
|
1058
|
+
already-stored future rows crossing into the range are dirtied even though
|
|
1059
|
+
no mutation sequence moved.
|
|
1060
|
+
"""
|
|
1061
|
+
_assert_owner()
|
|
1062
|
+
if (
|
|
1063
|
+
range_start.tzinfo is None or range_start.utcoffset() is None
|
|
1064
|
+
or range_end.tzinfo is None or range_end.utcoffset() is None
|
|
1065
|
+
or range_end <= range_start
|
|
1066
|
+
):
|
|
1067
|
+
raise ValueError("Codex accounting cache range must be aware and ordered")
|
|
1068
|
+
current_seq = _codex_accounting_mutation_seq(cache_conn)
|
|
1069
|
+
state = _CODEX_ACCOUNTING_CACHE_STATE
|
|
1070
|
+
prior_cached_entries = tuple(state.get("entries", ())) if state else ()
|
|
1071
|
+
cold = (
|
|
1072
|
+
current_seq is None
|
|
1073
|
+
or not state
|
|
1074
|
+
or state.get("extra") != extra_signature
|
|
1075
|
+
or state.get("start") != range_start
|
|
1076
|
+
or range_end < state.get("end", range_end)
|
|
1077
|
+
or current_seq < int(state.get("seq", 0))
|
|
1078
|
+
)
|
|
1079
|
+
dirty: set[tuple[str, str]] = set()
|
|
1080
|
+
if not cold:
|
|
1081
|
+
last_seq = int(state["seq"])
|
|
1082
|
+
if current_seq > last_seq:
|
|
1083
|
+
try:
|
|
1084
|
+
changes = tuple(cache_conn.execute(
|
|
1085
|
+
"SELECT mutation_seq, change_kind, source_root_key, source_path "
|
|
1086
|
+
"FROM codex_accounting_change_log "
|
|
1087
|
+
"WHERE mutation_seq > ? ORDER BY mutation_seq, seq",
|
|
1088
|
+
(last_seq,),
|
|
1089
|
+
))
|
|
1090
|
+
except sqlite3.Error:
|
|
1091
|
+
cold = True
|
|
1092
|
+
else:
|
|
1093
|
+
if (
|
|
1094
|
+
not changes
|
|
1095
|
+
or int(changes[0][0]) > last_seq + 1
|
|
1096
|
+
or int(changes[-1][0]) != current_seq
|
|
1097
|
+
or any(row[1] == "full" for row in changes)
|
|
1098
|
+
):
|
|
1099
|
+
cold = True
|
|
1100
|
+
else:
|
|
1101
|
+
dirty.update(
|
|
1102
|
+
(str(row[2] or ""), str(row[3]))
|
|
1103
|
+
for row in changes
|
|
1104
|
+
if row[1] == "path" and row[3]
|
|
1105
|
+
)
|
|
1106
|
+
prior_end = state.get("end")
|
|
1107
|
+
if not cold and isinstance(prior_end, dt.datetime) and range_end > prior_end:
|
|
1108
|
+
try:
|
|
1109
|
+
dirty.update(
|
|
1110
|
+
(str(root or ""), str(path))
|
|
1111
|
+
for root, path in cache_conn.execute(
|
|
1112
|
+
"SELECT DISTINCT source_root_key, source_path "
|
|
1113
|
+
"FROM codex_session_entries "
|
|
1114
|
+
"WHERE timestamp_utc >= ? AND timestamp_utc < ?",
|
|
1115
|
+
(prior_end.astimezone(dt.timezone.utc).isoformat(),
|
|
1116
|
+
range_end.astimezone(dt.timezone.utc).isoformat()),
|
|
1117
|
+
)
|
|
1118
|
+
)
|
|
1119
|
+
except sqlite3.Error:
|
|
1120
|
+
cold = True
|
|
1121
|
+
# The qualified path loader emits two SQL parameters and one OR arm per
|
|
1122
|
+
# identity. Beyond this bound a cold indexed range read is both safer
|
|
1123
|
+
# and cheaper than approaching SQLite's variable/expression limits.
|
|
1124
|
+
if not cold and len(dirty) > _CODEX_ACCOUNTING_MAX_DIRTY_PATHS:
|
|
1125
|
+
cold = True
|
|
1126
|
+
|
|
1127
|
+
if cold:
|
|
1128
|
+
entries = tuple(sorted(tuple(load_all()), key=order_key))
|
|
1129
|
+
if finalize_entries is not None:
|
|
1130
|
+
entries = tuple(finalize_entries(entries, None))
|
|
1131
|
+
accounts = tuple(sorted({
|
|
1132
|
+
str(account_of(entry))
|
|
1133
|
+
for entry in (*prior_cached_entries, *entries)
|
|
1134
|
+
}))
|
|
1135
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
1136
|
+
_CODEX_ACCOUNTING_CACHE_STATE.update({
|
|
1137
|
+
"entries": entries,
|
|
1138
|
+
"seq": 0 if current_seq is None else current_seq,
|
|
1139
|
+
"start": range_start,
|
|
1140
|
+
"end": range_end,
|
|
1141
|
+
"extra": extra_signature,
|
|
1142
|
+
})
|
|
1143
|
+
return CodexAccountingCacheResult(
|
|
1144
|
+
entries, (), accounts, True, prior_cached_entries, entries,
|
|
1145
|
+
)
|
|
1146
|
+
|
|
1147
|
+
prior_entries = tuple(state["entries"])
|
|
1148
|
+
if not dirty:
|
|
1149
|
+
state["seq"] = current_seq
|
|
1150
|
+
state["end"] = range_end
|
|
1151
|
+
return CodexAccountingCacheResult(prior_entries, (), (), False)
|
|
1152
|
+
|
|
1153
|
+
dirty_paths = tuple(sorted(dirty))
|
|
1154
|
+
prior_dirty = tuple(
|
|
1155
|
+
entry for entry in prior_entries if path_of(entry) in dirty
|
|
1156
|
+
)
|
|
1157
|
+
replacements = tuple(load_paths(dirty_paths))
|
|
1158
|
+
entries = tuple(sorted(
|
|
1159
|
+
(
|
|
1160
|
+
*(entry for entry in prior_entries if path_of(entry) not in dirty),
|
|
1161
|
+
*replacements,
|
|
1162
|
+
),
|
|
1163
|
+
key=order_key,
|
|
1164
|
+
))
|
|
1165
|
+
if finalize_entries is not None:
|
|
1166
|
+
entries = tuple(finalize_entries(entries, prior_entries))
|
|
1167
|
+
entry_identity = identity_of or order_key
|
|
1168
|
+
prior_by_identity = {entry_identity(entry): entry for entry in prior_dirty}
|
|
1169
|
+
replacement_by_identity = {
|
|
1170
|
+
entry_identity(entry): entry for entry in replacements
|
|
1171
|
+
}
|
|
1172
|
+
changed_old = tuple(
|
|
1173
|
+
entry for identity, entry in prior_by_identity.items()
|
|
1174
|
+
if identity not in replacement_by_identity
|
|
1175
|
+
or replacement_by_identity[identity] != entry
|
|
1176
|
+
)
|
|
1177
|
+
changed_new = tuple(
|
|
1178
|
+
entry for identity, entry in replacement_by_identity.items()
|
|
1179
|
+
if identity not in prior_by_identity
|
|
1180
|
+
or prior_by_identity[identity] != entry
|
|
1181
|
+
)
|
|
1182
|
+
dirty_accounts = tuple(sorted(
|
|
1183
|
+
{str(account_of(entry)) for entry in (*changed_old, *changed_new)}
|
|
1184
|
+
))
|
|
1185
|
+
state["entries"] = entries
|
|
1186
|
+
state["seq"] = current_seq
|
|
1187
|
+
state["end"] = range_end
|
|
1188
|
+
return CodexAccountingCacheResult(
|
|
1189
|
+
entries, dirty_paths, dirty_accounts, False, changed_old, changed_new,
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
|
|
963
1193
|
def build_cached_sessions(
|
|
964
1194
|
*,
|
|
965
1195
|
cache_conn: sqlite3.Connection,
|
|
@@ -1370,14 +1600,13 @@ def session_files_sig(cache_conn) -> "tuple[int, int]":
|
|
|
1370
1600
|
moves. Returns ``(0, 0)`` on a missing table (fresh DB) so callers never
|
|
1371
1601
|
raise.
|
|
1372
1602
|
|
|
1373
|
-
#271 §9d rider (
|
|
1374
|
-
|
|
1603
|
+
#271 §9d rider (tightened by #567): this ``(COUNT(*), MAX(rowid))`` leg
|
|
1604
|
+
does NOT by itself catch the in-place ``ON CONFLICT(path) DO UPDATE SET
|
|
1375
1605
|
project_path = COALESCE(...)`` attribution backfill — that UPDATE preserves
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
attribution move never both slips this signal and leaves the cache stale.
|
|
1606
|
+
both values. `_ensure_session_files_row` therefore stamps the joined
|
|
1607
|
+
`session_entries` rows with a fresh mutation sequence when it fills either
|
|
1608
|
+
identity column. The envelope's mutation-sequence reconcile catches that
|
|
1609
|
+
pure metadata move even when no new entry is ingested.
|
|
1381
1610
|
"""
|
|
1382
1611
|
try:
|
|
1383
1612
|
row = cache_conn.execute(
|
|
@@ -5,7 +5,7 @@ import datetime as dt
|
|
|
5
5
|
import hashlib
|
|
6
6
|
import math
|
|
7
7
|
from collections import defaultdict
|
|
8
|
-
from dataclasses import dataclass, replace
|
|
8
|
+
from dataclasses import dataclass, field, replace
|
|
9
9
|
from typing import Generic, Iterable, Literal, TypeVar
|
|
10
10
|
|
|
11
11
|
|
|
@@ -79,6 +79,11 @@ class QualifiedCodexEntry:
|
|
|
79
79
|
# contributes to. The default is the reserved `unattributed` sentinel, so
|
|
80
80
|
# every existing constructor (and the whole CLI path) is unchanged.
|
|
81
81
|
account_key: str = "unattributed"
|
|
82
|
+
# Internal stable ordering identity from cache.db. It is deliberately
|
|
83
|
+
# excluded from equality and representation: provider outputs do not carry
|
|
84
|
+
# it, while #582's dirty-path replacement needs the original SQL tie-break
|
|
85
|
+
# when rows share timestamp/root/conversation.
|
|
86
|
+
cache_entry_id: int = field(default=0, compare=False, repr=False)
|
|
82
87
|
|
|
83
88
|
|
|
84
89
|
def emitted_project_label(entry: QualifiedCodexEntry) -> str:
|
|
@@ -205,7 +210,20 @@ class TokenTotals:
|
|
|
205
210
|
|
|
206
211
|
|
|
207
212
|
def _totals(entries: Iterable[QualifiedCodexEntry]) -> TokenTotals:
|
|
208
|
-
|
|
213
|
+
# #566 §5.1 item 7. `TokenTotals` carries no label, and nothing this
|
|
214
|
+
# function returns can observe one, so the collision-safe allocation this
|
|
215
|
+
# line used to run was dead work: it re-created every entry through
|
|
216
|
+
# `dataclasses.replace` purely to sum six numeric fields off the copies.
|
|
217
|
+
# On the maintainer's store the 148 calls from here drove 775,568
|
|
218
|
+
# `dataclasses.replace` calls and about 1.9s of every build.
|
|
219
|
+
#
|
|
220
|
+
# The allocation that MATTERS still runs, once, in
|
|
221
|
+
# `build_codex_project_result`, over the complete population, before any
|
|
222
|
+
# subset is taken -- which is the only place it can be correct. Guarding
|
|
223
|
+
# this call on "the subset holds more than one project identity" would
|
|
224
|
+
# have kept nearly all of the cost, because the expensive callers are the
|
|
225
|
+
# per-block totals over a population spanning every project.
|
|
226
|
+
values = tuple(entries)
|
|
209
227
|
return TokenTotals(
|
|
210
228
|
input_tokens=sum(entry.input_tokens for entry in values),
|
|
211
229
|
cached_input_tokens=sum(entry.cached_input_tokens for entry in values),
|
package/bin/cctally
CHANGED
|
@@ -399,6 +399,7 @@ _lib_budget = _load_sibling("_lib_budget")
|
|
|
399
399
|
BudgetInputs = _lib_budget.BudgetInputs
|
|
400
400
|
BudgetStatus = _lib_budget.BudgetStatus
|
|
401
401
|
compute_budget_status = _lib_budget.compute_budget_status
|
|
402
|
+
budget_status_payload = _lib_budget.budget_status_payload
|
|
402
403
|
project_linear = _lib_budget.project_linear
|
|
403
404
|
calendar_month_window = _lib_budget.calendar_month_window
|
|
404
405
|
calendar_week_window = _lib_budget.calendar_week_window
|
|
@@ -1378,6 +1379,11 @@ _empty_dashboard_snapshot = _cctally_dashboard._empty_dashboard_snapshot
|
|
|
1378
1379
|
# which is also the seam its fold-failure tests monkeypatch.
|
|
1379
1380
|
resolve_shared_range = _cctally_dashboard.resolve_shared_range
|
|
1380
1381
|
iter_shared_range_entries = _cctally_dashboard.iter_shared_range_entries
|
|
1382
|
+
# #556 S5 §3.7: the Claude budget cost-event carrier rebuilds the same
|
|
1383
|
+
# `UsageEntry` these rows already feed, so the 1-hour cache-write split (#195)
|
|
1384
|
+
# is priced through the one `claude_usage_dict` chokepoint rather than a second
|
|
1385
|
+
# hand-rolled row unpack.
|
|
1386
|
+
_shared_range_row_to_usage_entry = _cctally_dashboard._shared_range_row_to_usage_entry
|
|
1381
1387
|
fold_projects_over_range = _cctally_dashboard.fold_projects_over_range
|
|
1382
1388
|
fold_daily_over_range = _cctally_dashboard.fold_daily_over_range
|
|
1383
1389
|
materialise_daily_calendar = _cctally_dashboard.materialise_daily_calendar
|
|
@@ -1385,6 +1391,12 @@ build_daily_aggregate_rows = _cctally_dashboard.build_daily_aggregate_rows
|
|
|
1385
1391
|
daily_panel_row_to_wire = _cctally_dashboard.daily_panel_row_to_wire
|
|
1386
1392
|
build_project_aggregate_rows = _cctally_dashboard.build_project_aggregate_rows
|
|
1387
1393
|
legacy_project_labels = _cctally_dashboard.legacy_project_labels
|
|
1394
|
+
build_cached_claude_range_aggregates = (
|
|
1395
|
+
_cctally_dashboard.build_cached_claude_range_aggregates
|
|
1396
|
+
)
|
|
1397
|
+
reset_claude_range_aggregate_memo = (
|
|
1398
|
+
_cctally_dashboard.reset_claude_range_aggregate_memo
|
|
1399
|
+
)
|
|
1388
1400
|
# _iso_z is NOT bound here anymore — the former dashboard-then-forecast
|
|
1389
1401
|
# double-bind collapses to a single canonical bind below (#279 S6 W4).
|
|
1390
1402
|
# Projects panel + modal (spec 2026-05-19-projects-panel-design.md).
|