cctally 1.97.0 → 1.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -4
- package/bin/_cctally_account.py +925 -0
- package/bin/_cctally_cache.py +829 -31
- package/bin/_cctally_core.py +52 -0
- package/bin/_cctally_dashboard.py +960 -55
- package/bin/_cctally_dashboard_envelope.py +43 -23
- package/bin/_cctally_dashboard_share.py +50 -1
- package/bin/_cctally_dashboard_sources.py +1580 -131
- package/bin/_cctally_db.py +810 -34
- package/bin/_cctally_doctor.py +184 -1
- package/bin/_cctally_journal.py +732 -76
- package/bin/_cctally_parser.py +65 -0
- package/bin/_cctally_quota.py +896 -17
- package/bin/_cctally_rederive.py +157 -5
- package/bin/_cctally_source_analytics.py +60 -6
- package/bin/_cctally_tui.py +811 -67
- package/bin/_lib_aggregators.py +117 -11
- package/bin/_lib_alert_axes.py +8 -3
- package/bin/_lib_budget.py +60 -0
- package/bin/_lib_codex_window_attribution.py +259 -0
- package/bin/_lib_dashboard_sources.py +488 -19
- package/bin/_lib_doctor.py +71 -0
- package/bin/_lib_journal.py +212 -0
- package/bin/_lib_jsonl.py +6 -0
- package/bin/_lib_rederive.py +8 -0
- package/bin/_lib_snapshot_cache.py +248 -7
- package/bin/_lib_source_analytics.py +20 -2
- package/bin/cctally +25 -0
- package/dashboard/static/assets/index-Bcbm-DNP.js +97 -0
- package/dashboard/static/assets/index-hJP4wlIO.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-BSESoPIK.css +0 -1
- package/dashboard/static/assets/index-DgsMz5hA.js +0 -97
package/bin/_lib_doctor.py
CHANGED
|
@@ -3351,6 +3351,75 @@ def _check_accounts_codex_reset_anchors(s: DoctorState) -> CheckResult:
|
|
|
3351
3351
|
)
|
|
3352
3352
|
|
|
3353
3353
|
|
|
3354
|
+
#: The `accounts.codex_window_attribution` conditions, in the order the summary
|
|
3355
|
+
#: names them. Each is resolved by retracting or re-asserting, which is why the
|
|
3356
|
+
#: leg is WARN in every case rather than FAIL: none of them is a broken install.
|
|
3357
|
+
_WINDOW_ATTRIBUTION_CONDITIONS: tuple[tuple[str, str], ...] = (
|
|
3358
|
+
("cursor_behind", "the derived index is behind the journal"),
|
|
3359
|
+
("dormant", "dormant assertion(s) matching no current window group"),
|
|
3360
|
+
("split", "assertion(s) matching more than one group after a split"),
|
|
3361
|
+
("conflicting", "assertion(s) in conflict with native evidence or each other"),
|
|
3362
|
+
("model_scoped", "assertion(s) over a window that is not account weekly quota"),
|
|
3363
|
+
("unrecoverable_baselines",
|
|
3364
|
+
"rollout file(s) whose per-file decision cannot be recovered"),
|
|
3365
|
+
)
|
|
3366
|
+
|
|
3367
|
+
|
|
3368
|
+
def _check_accounts_codex_window_attribution(s: DoctorState) -> CheckResult:
|
|
3369
|
+
"""Operator attribution of recorded Codex windows (#500 §9).
|
|
3370
|
+
|
|
3371
|
+
WARN in every case. The four spec conditions are a stale derived cursor
|
|
3372
|
+
(attribution is being UNDER-applied), a dormant assertion, a split
|
|
3373
|
+
assertion, and an assertion in conflict; the fifth and sixth are a
|
|
3374
|
+
model-scoped window an assertion can never file as account weekly quota, and
|
|
3375
|
+
a rollout file whose per-file baseline cannot be recovered — which matters
|
|
3376
|
+
because a retraction restores spend to that baseline, so an unrecoverable
|
|
3377
|
+
one is silently indistinguishable from "no decision was ever made".
|
|
3378
|
+
"""
|
|
3379
|
+
st = (s.accounts_state or {}).get("codex_window_attribution")
|
|
3380
|
+
if not isinstance(st, dict):
|
|
3381
|
+
# No cache.db, a cache too old to carry the derived table, or a probe
|
|
3382
|
+
# this run declined. Nothing to under-apply, so nothing to report.
|
|
3383
|
+
return CheckResult(
|
|
3384
|
+
id="accounts.codex_window_attribution",
|
|
3385
|
+
title="Codex window attribution", severity="ok",
|
|
3386
|
+
summary="no operator window attribution recorded",
|
|
3387
|
+
remediation=None, details={"active": 0, "retracted": 0},
|
|
3388
|
+
)
|
|
3389
|
+
details = dict(st)
|
|
3390
|
+
findings = []
|
|
3391
|
+
for name, phrase in _WINDOW_ATTRIBUTION_CONDITIONS:
|
|
3392
|
+
value = st.get(name)
|
|
3393
|
+
if name == "cursor_behind":
|
|
3394
|
+
if value:
|
|
3395
|
+
findings.append(phrase)
|
|
3396
|
+
continue
|
|
3397
|
+
try:
|
|
3398
|
+
count = int(value or 0)
|
|
3399
|
+
except (TypeError, ValueError):
|
|
3400
|
+
count = 0
|
|
3401
|
+
if count > 0:
|
|
3402
|
+
findings.append(f"{count} {phrase}")
|
|
3403
|
+
if not findings:
|
|
3404
|
+
return CheckResult(
|
|
3405
|
+
id="accounts.codex_window_attribution",
|
|
3406
|
+
title="Codex window attribution", severity="ok",
|
|
3407
|
+
summary=(f"{int(st.get('active') or 0)} active window "
|
|
3408
|
+
"attribution(s), all applying cleanly"),
|
|
3409
|
+
remediation=None, details=details,
|
|
3410
|
+
)
|
|
3411
|
+
return CheckResult(
|
|
3412
|
+
id="accounts.codex_window_attribution",
|
|
3413
|
+
title="Codex window attribution", severity="warn",
|
|
3414
|
+
summary="; ".join(findings),
|
|
3415
|
+
remediation=(
|
|
3416
|
+
"Run `cctally account attribute <ref> --since <iso> --retract` to "
|
|
3417
|
+
"clear an assertion that no longer applies, or re-assert it over "
|
|
3418
|
+
"the group it should name"),
|
|
3419
|
+
details=details,
|
|
3420
|
+
)
|
|
3421
|
+
|
|
3422
|
+
|
|
3354
3423
|
_CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...] = (
|
|
3355
3424
|
("install", "Install", (
|
|
3356
3425
|
("install.mode", "_check_install_dev_mode"),
|
|
@@ -3423,6 +3492,8 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
|
|
|
3423
3492
|
("accounts.freshness", "_check_accounts_freshness"),
|
|
3424
3493
|
("accounts.attribution", "_check_accounts_attribution"),
|
|
3425
3494
|
("accounts.codex_reset_anchors", "_check_accounts_codex_reset_anchors"),
|
|
3495
|
+
("accounts.codex_window_attribution",
|
|
3496
|
+
"_check_accounts_codex_window_attribution"),
|
|
3426
3497
|
)),
|
|
3427
3498
|
("pricing", "Pricing", (
|
|
3428
3499
|
("pricing.coverage", "_check_pricing_coverage"),
|
package/bin/_lib_journal.py
CHANGED
|
@@ -1308,6 +1308,218 @@ def make_codex_file_account(
|
|
|
1308
1308
|
return make_op(at=at, src="codex-file-account", payload=payload)
|
|
1309
1309
|
|
|
1310
1310
|
|
|
1311
|
+
#: Only account-level weekly Codex quota is attributable (#500 spec §5). A 5h
|
|
1312
|
+
#: window and a model-scoped pool are out of scope, and the builder refuses
|
|
1313
|
+
#: rather than recording an assertion the overlay would always suppress.
|
|
1314
|
+
ACCOUNT_WEEKLY_WINDOW_MINUTES = 10_080
|
|
1315
|
+
|
|
1316
|
+
#: The sentinel is never an assertable subject and is never written into a
|
|
1317
|
+
#: payload (the two-shaped stamp rule, docs/accounts-gotchas.md).
|
|
1318
|
+
_ATTRIBUTION_SENTINEL = "unattributed"
|
|
1319
|
+
|
|
1320
|
+
WINDOW_ATTRIBUTION_KIND = "codex_window_attribution"
|
|
1321
|
+
WINDOW_ATTRIBUTION_RETRACT_KIND = "codex_window_attribution_retract"
|
|
1322
|
+
WINDOW_ATTRIBUTION_SRC = "account-attribute"
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _sorted_unique_strings(values, what: str) -> list:
|
|
1326
|
+
"""Sorted, de-duplicated, non-empty strings, or ``ValueError``.
|
|
1327
|
+
|
|
1328
|
+
A bare string is refused rather than iterated: ``"abc"`` would otherwise
|
|
1329
|
+
silently become three one-character witnesses. Types are checked BEFORE the
|
|
1330
|
+
sort, so a non-string member raises ``ValueError`` rather than the
|
|
1331
|
+
``TypeError`` a mixed-type ``sorted`` would raise.
|
|
1332
|
+
"""
|
|
1333
|
+
if isinstance(values, (str, bytes)):
|
|
1334
|
+
raise ValueError(f"{what} must be a sequence of strings, not a string")
|
|
1335
|
+
try:
|
|
1336
|
+
items = list(values)
|
|
1337
|
+
except TypeError as exc:
|
|
1338
|
+
raise ValueError(f"{what} must be a sequence of strings") from exc
|
|
1339
|
+
for item in items:
|
|
1340
|
+
if not isinstance(item, str) or not item:
|
|
1341
|
+
raise ValueError(f"every {what} entry must be a non-empty string")
|
|
1342
|
+
return sorted(set(items))
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def _normalize_attribution_instant(at: str) -> str:
|
|
1346
|
+
"""One ISO spelling for a #500 assertion's `at` (review finding F9).
|
|
1347
|
+
|
|
1348
|
+
`make_op` stores the caller's string verbatim, and the active read orders
|
|
1349
|
+
assertions with `ORDER BY asserted_at_utc ASC` — a LEXICOGRAPHIC sort over
|
|
1350
|
+
that stored text. Lexicographic order is chronological order only while
|
|
1351
|
+
every value uses one spelling: `2026-08-14T12:00:00+00:00` sorts before
|
|
1352
|
+
`2026-08-14T06:00:00Z` because `+` (0x2B) precedes `Z` (0x5A), so a single
|
|
1353
|
+
offset-form assertion would silently take precedence over every later
|
|
1354
|
+
Z-form one. Normalizing HERE, at the one chokepoint both builders pass
|
|
1355
|
+
through, is what makes the cheap SQL ordering correct.
|
|
1356
|
+
|
|
1357
|
+
MICROSECOND precision with a `Z` suffix, and the six fractional digits are
|
|
1358
|
+
written ALWAYS, even when they are zero (review round 2, finding R2-1).
|
|
1359
|
+
Two separate rules meet here and both require the fixed width.
|
|
1360
|
+
|
|
1361
|
+
Truncating to seconds is a correctness defect, not a cosmetic one. The
|
|
1362
|
+
normalized value is what `content_id` digests, so two assertions inside one
|
|
1363
|
+
wall-clock second with otherwise-identical payloads collapse to the SAME
|
|
1364
|
+
`op_id`. Spec §7.2's assert -> retract -> re-assert then loses its last
|
|
1365
|
+
step: the re-assertion carries the id the tombstone already names, so
|
|
1366
|
+
`INSERT OR IGNORE` drops it against the retracted row and the operator's
|
|
1367
|
+
second assertion vanishes with no error. §7.2 requires a re-assertion to
|
|
1368
|
+
carry an id no earlier tombstone names, and sub-second precision is what
|
|
1369
|
+
supplies it.
|
|
1370
|
+
|
|
1371
|
+
Emitting the fraction only when non-zero would break the ordering rule
|
|
1372
|
+
above: `.` (0x2E) precedes `Z` (0x5A), so `2026-08-14T00:00:05.500000Z`
|
|
1373
|
+
would sort BEFORE `2026-08-14T00:00:05Z` — a fractional value ahead of the
|
|
1374
|
+
whole second it follows. One fixed-width spelling is what keeps
|
|
1375
|
+
lexicographic order equal to chronological order.
|
|
1376
|
+
|
|
1377
|
+
A naive value is refused rather than assumed to be UTC: the assertion is
|
|
1378
|
+
durable, and guessing a zone for it is not recoverable.
|
|
1379
|
+
"""
|
|
1380
|
+
if not isinstance(at, str) or not at:
|
|
1381
|
+
raise ValueError("at must be a non-empty ISO instant")
|
|
1382
|
+
try:
|
|
1383
|
+
parsed = dt.datetime.fromisoformat(at.replace("Z", "+00:00"))
|
|
1384
|
+
except ValueError as exc:
|
|
1385
|
+
raise ValueError(f"at is not an ISO instant: {at!r}") from exc
|
|
1386
|
+
if parsed.tzinfo is None:
|
|
1387
|
+
raise ValueError(f"at must carry a timezone offset: {at!r}")
|
|
1388
|
+
return parsed.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _window_attribution_payload(
|
|
1392
|
+
kind: str,
|
|
1393
|
+
*,
|
|
1394
|
+
account_key: str,
|
|
1395
|
+
source_root_key: str,
|
|
1396
|
+
logical_limit_key: str,
|
|
1397
|
+
observed_slot: str,
|
|
1398
|
+
window_minutes: int,
|
|
1399
|
+
raw_resets_at_utc,
|
|
1400
|
+
canonical_resets_at_utc: str,
|
|
1401
|
+
) -> dict:
|
|
1402
|
+
"""Shared payload for the two #500 operator-attribution op kinds.
|
|
1403
|
+
|
|
1404
|
+
The binding is the four normalized axes plus the tolerance-connected
|
|
1405
|
+
component WITNESSED by ``raw_resets_at_utc``. The canonical anchor is
|
|
1406
|
+
carried for audit and preview only and is NEVER matched on: it is
|
|
1407
|
+
population-dependent, because a later bridging observation can union two
|
|
1408
|
+
components and retire the earlier anchor (``_lib_quota`` component union),
|
|
1409
|
+
which is exactly why the canonical value is not journaled anywhere else.
|
|
1410
|
+
|
|
1411
|
+
The ``account_key`` here is the SUBJECT of the assertion — the account the
|
|
1412
|
+
operator says the window belongs to — not the two-shaped stamp naming which
|
|
1413
|
+
account wrote the record. Both kinds are registered in
|
|
1414
|
+
``_ACCOUNTS_MACHINERY_KINDS``, but NOT because the legacy normalizer would
|
|
1415
|
+
otherwise mistake this field for a missing stamp (review finding F7) — it
|
|
1416
|
+
would not, being gated on ``_REAL_ACCOUNT_EVT_OP_KINDS``, which these kinds
|
|
1417
|
+
were never in. Registration is what puts the kinds into the re-derive
|
|
1418
|
+
planner's ``op_kinds`` set, which makes their
|
|
1419
|
+
``_lib_rederive._OP_CLASSIFICATIONS`` entries mandatory.
|
|
1420
|
+
"""
|
|
1421
|
+
if not isinstance(account_key, str) or not account_key:
|
|
1422
|
+
raise ValueError("account_key must be a non-empty string")
|
|
1423
|
+
if account_key == _ATTRIBUTION_SENTINEL:
|
|
1424
|
+
raise ValueError("the unattributed sentinel is not an assertable subject")
|
|
1425
|
+
for name, value in (("source_root_key", source_root_key),
|
|
1426
|
+
("logical_limit_key", logical_limit_key),
|
|
1427
|
+
("observed_slot", observed_slot),
|
|
1428
|
+
("canonical_resets_at_utc", canonical_resets_at_utc)):
|
|
1429
|
+
if not isinstance(value, str) or not value:
|
|
1430
|
+
raise ValueError(f"{name} must be a non-empty string")
|
|
1431
|
+
if (not isinstance(window_minutes, int)
|
|
1432
|
+
or isinstance(window_minutes, bool)
|
|
1433
|
+
or window_minutes != ACCOUNT_WEEKLY_WINDOW_MINUTES):
|
|
1434
|
+
raise ValueError(
|
|
1435
|
+
f"window_minutes must be {ACCOUNT_WEEKLY_WINDOW_MINUTES} "
|
|
1436
|
+
"(account weekly quota)")
|
|
1437
|
+
witnesses = _sorted_unique_strings(raw_resets_at_utc, "raw reset witness")
|
|
1438
|
+
if not witnesses:
|
|
1439
|
+
raise ValueError("raw_resets_at_utc must carry at least one witness")
|
|
1440
|
+
return {
|
|
1441
|
+
"kind": kind,
|
|
1442
|
+
"source": "codex",
|
|
1443
|
+
"account_key": account_key,
|
|
1444
|
+
"source_root_key": source_root_key,
|
|
1445
|
+
"logical_limit_key": logical_limit_key,
|
|
1446
|
+
"observed_slot": observed_slot,
|
|
1447
|
+
"window_minutes": window_minutes,
|
|
1448
|
+
"raw_resets_at_utc": witnesses,
|
|
1449
|
+
"canonical_resets_at_utc": canonical_resets_at_utc,
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def make_codex_window_attribution(
|
|
1454
|
+
at: str,
|
|
1455
|
+
*,
|
|
1456
|
+
account_key: str,
|
|
1457
|
+
source_root_key: str,
|
|
1458
|
+
logical_limit_key: str,
|
|
1459
|
+
observed_slot: str,
|
|
1460
|
+
window_minutes: int,
|
|
1461
|
+
raw_resets_at_utc,
|
|
1462
|
+
canonical_resets_at_utc: str,
|
|
1463
|
+
) -> dict:
|
|
1464
|
+
"""Build a ``codex_window_attribution`` op — the operator's durable
|
|
1465
|
+
assertion that one physical Codex quota window group belongs to one account
|
|
1466
|
+
(#500 spec §5).
|
|
1467
|
+
|
|
1468
|
+
Ordinary ``content_id``, deliberately NOT a stable singleton id like the
|
|
1469
|
+
accounts cutover's: a fixed id would make assert -> retract -> reassert
|
|
1470
|
+
impossible, because the second assertion would collide with the first.
|
|
1471
|
+
"""
|
|
1472
|
+
payload = _window_attribution_payload(
|
|
1473
|
+
WINDOW_ATTRIBUTION_KIND,
|
|
1474
|
+
account_key=account_key,
|
|
1475
|
+
source_root_key=source_root_key,
|
|
1476
|
+
logical_limit_key=logical_limit_key,
|
|
1477
|
+
observed_slot=observed_slot,
|
|
1478
|
+
window_minutes=window_minutes,
|
|
1479
|
+
raw_resets_at_utc=raw_resets_at_utc,
|
|
1480
|
+
canonical_resets_at_utc=canonical_resets_at_utc,
|
|
1481
|
+
)
|
|
1482
|
+
return make_op(at=_normalize_attribution_instant(at),
|
|
1483
|
+
src=WINDOW_ATTRIBUTION_SRC, payload=payload)
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def make_codex_window_attribution_retract(
|
|
1487
|
+
at: str,
|
|
1488
|
+
*,
|
|
1489
|
+
account_key: str,
|
|
1490
|
+
source_root_key: str,
|
|
1491
|
+
logical_limit_key: str,
|
|
1492
|
+
observed_slot: str,
|
|
1493
|
+
window_minutes: int,
|
|
1494
|
+
raw_resets_at_utc,
|
|
1495
|
+
canonical_resets_at_utc: str,
|
|
1496
|
+
retracted_assertion_ids,
|
|
1497
|
+
) -> dict:
|
|
1498
|
+
"""Build a ``codex_window_attribution_retract`` op (#500 spec §5, §7.2).
|
|
1499
|
+
|
|
1500
|
+
Targets specific assertion op IDs rather than the group, so an older
|
|
1501
|
+
tombstone can never suppress a later reassertion: the reassertion carries a
|
|
1502
|
+
new content id that no earlier tombstone names.
|
|
1503
|
+
"""
|
|
1504
|
+
payload = _window_attribution_payload(
|
|
1505
|
+
WINDOW_ATTRIBUTION_RETRACT_KIND,
|
|
1506
|
+
account_key=account_key,
|
|
1507
|
+
source_root_key=source_root_key,
|
|
1508
|
+
logical_limit_key=logical_limit_key,
|
|
1509
|
+
observed_slot=observed_slot,
|
|
1510
|
+
window_minutes=window_minutes,
|
|
1511
|
+
raw_resets_at_utc=raw_resets_at_utc,
|
|
1512
|
+
canonical_resets_at_utc=canonical_resets_at_utc,
|
|
1513
|
+
)
|
|
1514
|
+
targets = _sorted_unique_strings(
|
|
1515
|
+
retracted_assertion_ids, "retracted assertion id")
|
|
1516
|
+
if not targets:
|
|
1517
|
+
raise ValueError("retracted_assertion_ids must name at least one assertion")
|
|
1518
|
+
payload["retracted_assertion_ids"] = targets
|
|
1519
|
+
return make_op(at=_normalize_attribution_instant(at),
|
|
1520
|
+
src=WINDOW_ATTRIBUTION_SRC, payload=payload)
|
|
1521
|
+
|
|
1522
|
+
|
|
1311
1523
|
# --------------------------------------------------------------------------
|
|
1312
1524
|
# segment naming + canonical order
|
|
1313
1525
|
# --------------------------------------------------------------------------
|
package/bin/_lib_jsonl.py
CHANGED
|
@@ -81,6 +81,12 @@ class CodexEntry:
|
|
|
81
81
|
reasoning_output_tokens: int
|
|
82
82
|
total_tokens: int
|
|
83
83
|
source_path: str
|
|
84
|
+
# Cache-backed readers already compute the canonical price while loading.
|
|
85
|
+
# Direct JSONL/CLI readers leave this unset and retain historical pricing.
|
|
86
|
+
cost_usd: float | None = None
|
|
87
|
+
cache_entry_id: int = field(default=0, compare=False, repr=False)
|
|
88
|
+
source_root_key: str = field(default="", compare=False, repr=False)
|
|
89
|
+
conversation_key: str = field(default="", compare=False, repr=False)
|
|
84
90
|
|
|
85
91
|
|
|
86
92
|
def _entry_token_total(entry: "UsageEntry") -> int:
|
package/bin/_lib_rederive.py
CHANGED
|
@@ -105,6 +105,14 @@ _OP_CLASSIFICATIONS = {
|
|
|
105
105
|
# leg rather than into the claude-usage scratch index.
|
|
106
106
|
"codex_file_account": KindClassification(
|
|
107
107
|
"retained", "Codex file attribution decision is outside claude-usage"),
|
|
108
|
+
# #500: operator attribution of recorded Codex quota windows. `retained`,
|
|
109
|
+
# exactly like the `codex_file_account` precedent above — provider state
|
|
110
|
+
# OUTSIDE the claude-usage family, replayed by the Codex cache leg rather
|
|
111
|
+
# than into the claude-usage scratch index.
|
|
112
|
+
"codex_window_attribution": KindClassification(
|
|
113
|
+
"retained", "operator Codex window attribution is outside claude-usage"),
|
|
114
|
+
"codex_window_attribution_retract": KindClassification(
|
|
115
|
+
"retained", "operator Codex window retraction is outside claude-usage"),
|
|
108
116
|
}
|
|
109
117
|
|
|
110
118
|
|
|
@@ -91,6 +91,11 @@ class SnapshotSignature(NamedTuple):
|
|
|
91
91
|
# sequence supplies that missing identity leg; the stats digest arrives
|
|
92
92
|
# from the independently-committed quota/budget projection database.
|
|
93
93
|
codex_physical_mutation_seq: int = 0
|
|
94
|
+
# #582: path-ledger accounting changes must leave the snapshot idle path,
|
|
95
|
+
# even when MAX(id) and the broader physical counter stay flat. This leg
|
|
96
|
+
# is internal dispatch identity only; published data-version bytes retain
|
|
97
|
+
# their existing contract.
|
|
98
|
+
codex_accounting_mutation_seq: int = 0
|
|
94
99
|
codex_stats_digest: str = ""
|
|
95
100
|
# #341 finding 9: a digest of the account registry + the providers' on-disk
|
|
96
101
|
# identity-file/active-account state. Empty for every <=1-account install (no
|
|
@@ -106,6 +111,16 @@ class SnapshotSignature(NamedTuple):
|
|
|
106
111
|
# Codex mutation happens along. Empty when nothing is owed (the writer
|
|
107
112
|
# DELETEs the key at zero), so a fully-ingested store is byte-neutral.
|
|
108
113
|
codex_ingest_backlog_sig: str = ""
|
|
114
|
+
# #556 S3 §2.9: the Claude twin of `codex_stats_digest`. The stats legs
|
|
115
|
+
# above are `MAX(id)` over the two weekly snapshot tables plus the
|
|
116
|
+
# reset-event change signal, and a Claude milestone INSERT or an
|
|
117
|
+
# `alerted_at` arming UPDATE touches none of them — measured: inserting a
|
|
118
|
+
# `budget_milestones` row with `vendor='claude'` left every other leg
|
|
119
|
+
# byte-identical. Without this leg a fired Claude alert could leave the
|
|
120
|
+
# idle path short-circuiting on a retained prior bundle. Unlike the two
|
|
121
|
+
# legs above, this one is a digest and is never empty: a store with no
|
|
122
|
+
# armed Claude alert carries a constant hash, not the empty string.
|
|
123
|
+
claude_stats_digest: str = ""
|
|
109
124
|
|
|
110
125
|
|
|
111
126
|
def _max_id(conn: sqlite3.Connection, table: str) -> int:
|
|
@@ -221,6 +236,7 @@ def compute_signature(
|
|
|
221
236
|
generation: int,
|
|
222
237
|
codex_stats_digest: str = "",
|
|
223
238
|
accounts_digest: str = "",
|
|
239
|
+
claude_stats_digest: str = "",
|
|
224
240
|
) -> SnapshotSignature:
|
|
225
241
|
"""Composite data-version signature across cache.db + stats.db (spec §3).
|
|
226
242
|
|
|
@@ -240,9 +256,13 @@ def compute_signature(
|
|
|
240
256
|
generation=int(generation),
|
|
241
257
|
entry_mutation_seq=_entry_mutation_seq(cache_conn),
|
|
242
258
|
codex_physical_mutation_seq=_codex_physical_mutation_seq(cache_conn),
|
|
259
|
+
codex_accounting_mutation_seq=(
|
|
260
|
+
_codex_accounting_mutation_seq(cache_conn) or 0
|
|
261
|
+
),
|
|
243
262
|
codex_stats_digest=str(codex_stats_digest),
|
|
244
263
|
accounts_digest=str(accounts_digest),
|
|
245
264
|
codex_ingest_backlog_sig=_codex_ingest_backlog_sig(cache_conn),
|
|
265
|
+
claude_stats_digest=str(claude_stats_digest),
|
|
246
266
|
)
|
|
247
267
|
|
|
248
268
|
|
|
@@ -948,6 +968,228 @@ def reset_session_cache_state() -> None:
|
|
|
948
968
|
_SESSION_LAST_SEEN.clear()
|
|
949
969
|
|
|
950
970
|
|
|
971
|
+
# === #582 — persistent Codex accounting rows by dirty physical path =========
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
@dataclass(frozen=True)
|
|
975
|
+
class CodexAccountingCacheResult:
|
|
976
|
+
"""One cold or incrementally refreshed accounting population."""
|
|
977
|
+
|
|
978
|
+
entries: tuple[object, ...]
|
|
979
|
+
dirty_paths: tuple[tuple[str, str], ...]
|
|
980
|
+
dirty_accounts: tuple[str, ...]
|
|
981
|
+
cold: bool
|
|
982
|
+
changed_old: tuple[object, ...] = ()
|
|
983
|
+
changed_new: tuple[object, ...] = ()
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
_CODEX_ACCOUNTING_CACHE_STATE: dict[str, object] = {}
|
|
987
|
+
_CODEX_ACCOUNTING_MAX_DIRTY_PATHS = 300
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def reset_codex_accounting_cache_state() -> None:
|
|
991
|
+
"""Drop #582's value-only Codex accounting cache and ledger cursor."""
|
|
992
|
+
_assert_owner()
|
|
993
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
def checkpoint_codex_accounting_cache_state() -> dict[str, object]:
|
|
997
|
+
"""Copy the value-only state so a failed source build can roll back."""
|
|
998
|
+
_assert_owner()
|
|
999
|
+
return dict(_CODEX_ACCOUNTING_CACHE_STATE)
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def restore_codex_accounting_cache_state(state: dict[str, object]) -> None:
|
|
1003
|
+
"""Restore a checkpoint after downstream source construction fails."""
|
|
1004
|
+
_assert_owner()
|
|
1005
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
1006
|
+
_CODEX_ACCOUNTING_CACHE_STATE.update(state)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def _codex_accounting_mutation_seq(conn: sqlite3.Connection) -> int | None:
|
|
1010
|
+
"""Return the dedicated ledger sequence, or ``None`` when unsupported."""
|
|
1011
|
+
try:
|
|
1012
|
+
row = conn.execute(
|
|
1013
|
+
"SELECT value FROM cache_meta "
|
|
1014
|
+
"WHERE key='codex_accounting_mutation_seq'"
|
|
1015
|
+
).fetchone()
|
|
1016
|
+
table = conn.execute(
|
|
1017
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' "
|
|
1018
|
+
"AND name='codex_accounting_change_log'"
|
|
1019
|
+
).fetchone()
|
|
1020
|
+
except sqlite3.Error:
|
|
1021
|
+
return None
|
|
1022
|
+
if table is None or row is None or row[0] is None:
|
|
1023
|
+
return None
|
|
1024
|
+
try:
|
|
1025
|
+
return int(row[0])
|
|
1026
|
+
except (TypeError, ValueError):
|
|
1027
|
+
return None
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def codex_accounting_cache_pending(conn: sqlite3.Connection) -> bool:
|
|
1031
|
+
"""Whether the process cache has not consumed the durable ledger head."""
|
|
1032
|
+
_assert_owner()
|
|
1033
|
+
current = _codex_accounting_mutation_seq(conn)
|
|
1034
|
+
if current is None or not _CODEX_ACCOUNTING_CACHE_STATE:
|
|
1035
|
+
return True
|
|
1036
|
+
return current != int(_CODEX_ACCOUNTING_CACHE_STATE.get("seq", -1))
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def build_cached_codex_accounting(
|
|
1040
|
+
*,
|
|
1041
|
+
cache_conn: sqlite3.Connection,
|
|
1042
|
+
range_start: dt.datetime,
|
|
1043
|
+
range_end: dt.datetime,
|
|
1044
|
+
extra_signature: object,
|
|
1045
|
+
load_all: "Callable[[], tuple[object, ...]]",
|
|
1046
|
+
load_paths: "Callable[[tuple[tuple[str, str], ...]], tuple[object, ...]]",
|
|
1047
|
+
path_of: "Callable[[object], tuple[str, str]]",
|
|
1048
|
+
account_of: "Callable[[object], str]",
|
|
1049
|
+
order_key: "Callable[[object], object]",
|
|
1050
|
+
identity_of: "Callable[[object], object] | None" = None,
|
|
1051
|
+
finalize_entries: "Callable[[tuple[object, ...], tuple[object, ...] | None], tuple[object, ...]] | None" = None,
|
|
1052
|
+
) -> CodexAccountingCacheResult:
|
|
1053
|
+
"""Cold-load once, then replace only ledger-dirty Codex file populations.
|
|
1054
|
+
|
|
1055
|
+
The cache retains plain immutable values and one integer cursor. A missing
|
|
1056
|
+
ledger, cursor gap, full marker, semantic-key change, range-start change,
|
|
1057
|
+
or clock regression goes cold. When the upper bound advances, paths with
|
|
1058
|
+
already-stored future rows crossing into the range are dirtied even though
|
|
1059
|
+
no mutation sequence moved.
|
|
1060
|
+
"""
|
|
1061
|
+
_assert_owner()
|
|
1062
|
+
if (
|
|
1063
|
+
range_start.tzinfo is None or range_start.utcoffset() is None
|
|
1064
|
+
or range_end.tzinfo is None or range_end.utcoffset() is None
|
|
1065
|
+
or range_end <= range_start
|
|
1066
|
+
):
|
|
1067
|
+
raise ValueError("Codex accounting cache range must be aware and ordered")
|
|
1068
|
+
current_seq = _codex_accounting_mutation_seq(cache_conn)
|
|
1069
|
+
state = _CODEX_ACCOUNTING_CACHE_STATE
|
|
1070
|
+
prior_cached_entries = tuple(state.get("entries", ())) if state else ()
|
|
1071
|
+
cold = (
|
|
1072
|
+
current_seq is None
|
|
1073
|
+
or not state
|
|
1074
|
+
or state.get("extra") != extra_signature
|
|
1075
|
+
or state.get("start") != range_start
|
|
1076
|
+
or range_end < state.get("end", range_end)
|
|
1077
|
+
or current_seq < int(state.get("seq", 0))
|
|
1078
|
+
)
|
|
1079
|
+
dirty: set[tuple[str, str]] = set()
|
|
1080
|
+
if not cold:
|
|
1081
|
+
last_seq = int(state["seq"])
|
|
1082
|
+
if current_seq > last_seq:
|
|
1083
|
+
try:
|
|
1084
|
+
changes = tuple(cache_conn.execute(
|
|
1085
|
+
"SELECT mutation_seq, change_kind, source_root_key, source_path "
|
|
1086
|
+
"FROM codex_accounting_change_log "
|
|
1087
|
+
"WHERE mutation_seq > ? ORDER BY mutation_seq, seq",
|
|
1088
|
+
(last_seq,),
|
|
1089
|
+
))
|
|
1090
|
+
except sqlite3.Error:
|
|
1091
|
+
cold = True
|
|
1092
|
+
else:
|
|
1093
|
+
if (
|
|
1094
|
+
not changes
|
|
1095
|
+
or int(changes[0][0]) > last_seq + 1
|
|
1096
|
+
or int(changes[-1][0]) != current_seq
|
|
1097
|
+
or any(row[1] == "full" for row in changes)
|
|
1098
|
+
):
|
|
1099
|
+
cold = True
|
|
1100
|
+
else:
|
|
1101
|
+
dirty.update(
|
|
1102
|
+
(str(row[2] or ""), str(row[3]))
|
|
1103
|
+
for row in changes
|
|
1104
|
+
if row[1] == "path" and row[3]
|
|
1105
|
+
)
|
|
1106
|
+
prior_end = state.get("end")
|
|
1107
|
+
if not cold and isinstance(prior_end, dt.datetime) and range_end > prior_end:
|
|
1108
|
+
try:
|
|
1109
|
+
dirty.update(
|
|
1110
|
+
(str(root or ""), str(path))
|
|
1111
|
+
for root, path in cache_conn.execute(
|
|
1112
|
+
"SELECT DISTINCT source_root_key, source_path "
|
|
1113
|
+
"FROM codex_session_entries "
|
|
1114
|
+
"WHERE timestamp_utc >= ? AND timestamp_utc < ?",
|
|
1115
|
+
(prior_end.astimezone(dt.timezone.utc).isoformat(),
|
|
1116
|
+
range_end.astimezone(dt.timezone.utc).isoformat()),
|
|
1117
|
+
)
|
|
1118
|
+
)
|
|
1119
|
+
except sqlite3.Error:
|
|
1120
|
+
cold = True
|
|
1121
|
+
# The qualified path loader emits two SQL parameters and one OR arm per
|
|
1122
|
+
# identity. Beyond this bound a cold indexed range read is both safer
|
|
1123
|
+
# and cheaper than approaching SQLite's variable/expression limits.
|
|
1124
|
+
if not cold and len(dirty) > _CODEX_ACCOUNTING_MAX_DIRTY_PATHS:
|
|
1125
|
+
cold = True
|
|
1126
|
+
|
|
1127
|
+
if cold:
|
|
1128
|
+
entries = tuple(sorted(tuple(load_all()), key=order_key))
|
|
1129
|
+
if finalize_entries is not None:
|
|
1130
|
+
entries = tuple(finalize_entries(entries, None))
|
|
1131
|
+
accounts = tuple(sorted({
|
|
1132
|
+
str(account_of(entry))
|
|
1133
|
+
for entry in (*prior_cached_entries, *entries)
|
|
1134
|
+
}))
|
|
1135
|
+
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
1136
|
+
_CODEX_ACCOUNTING_CACHE_STATE.update({
|
|
1137
|
+
"entries": entries,
|
|
1138
|
+
"seq": 0 if current_seq is None else current_seq,
|
|
1139
|
+
"start": range_start,
|
|
1140
|
+
"end": range_end,
|
|
1141
|
+
"extra": extra_signature,
|
|
1142
|
+
})
|
|
1143
|
+
return CodexAccountingCacheResult(
|
|
1144
|
+
entries, (), accounts, True, prior_cached_entries, entries,
|
|
1145
|
+
)
|
|
1146
|
+
|
|
1147
|
+
prior_entries = tuple(state["entries"])
|
|
1148
|
+
if not dirty:
|
|
1149
|
+
state["seq"] = current_seq
|
|
1150
|
+
state["end"] = range_end
|
|
1151
|
+
return CodexAccountingCacheResult(prior_entries, (), (), False)
|
|
1152
|
+
|
|
1153
|
+
dirty_paths = tuple(sorted(dirty))
|
|
1154
|
+
prior_dirty = tuple(
|
|
1155
|
+
entry for entry in prior_entries if path_of(entry) in dirty
|
|
1156
|
+
)
|
|
1157
|
+
replacements = tuple(load_paths(dirty_paths))
|
|
1158
|
+
entries = tuple(sorted(
|
|
1159
|
+
(
|
|
1160
|
+
*(entry for entry in prior_entries if path_of(entry) not in dirty),
|
|
1161
|
+
*replacements,
|
|
1162
|
+
),
|
|
1163
|
+
key=order_key,
|
|
1164
|
+
))
|
|
1165
|
+
if finalize_entries is not None:
|
|
1166
|
+
entries = tuple(finalize_entries(entries, prior_entries))
|
|
1167
|
+
entry_identity = identity_of or order_key
|
|
1168
|
+
prior_by_identity = {entry_identity(entry): entry for entry in prior_dirty}
|
|
1169
|
+
replacement_by_identity = {
|
|
1170
|
+
entry_identity(entry): entry for entry in replacements
|
|
1171
|
+
}
|
|
1172
|
+
changed_old = tuple(
|
|
1173
|
+
entry for identity, entry in prior_by_identity.items()
|
|
1174
|
+
if identity not in replacement_by_identity
|
|
1175
|
+
or replacement_by_identity[identity] != entry
|
|
1176
|
+
)
|
|
1177
|
+
changed_new = tuple(
|
|
1178
|
+
entry for identity, entry in replacement_by_identity.items()
|
|
1179
|
+
if identity not in prior_by_identity
|
|
1180
|
+
or prior_by_identity[identity] != entry
|
|
1181
|
+
)
|
|
1182
|
+
dirty_accounts = tuple(sorted(
|
|
1183
|
+
{str(account_of(entry)) for entry in (*changed_old, *changed_new)}
|
|
1184
|
+
))
|
|
1185
|
+
state["entries"] = entries
|
|
1186
|
+
state["seq"] = current_seq
|
|
1187
|
+
state["end"] = range_end
|
|
1188
|
+
return CodexAccountingCacheResult(
|
|
1189
|
+
entries, dirty_paths, dirty_accounts, False, changed_old, changed_new,
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
|
|
951
1193
|
def build_cached_sessions(
|
|
952
1194
|
*,
|
|
953
1195
|
cache_conn: sqlite3.Connection,
|
|
@@ -1358,14 +1600,13 @@ def session_files_sig(cache_conn) -> "tuple[int, int]":
|
|
|
1358
1600
|
moves. Returns ``(0, 0)`` on a missing table (fresh DB) so callers never
|
|
1359
1601
|
raise.
|
|
1360
1602
|
|
|
1361
|
-
#271 §9d rider (
|
|
1362
|
-
|
|
1603
|
+
#271 §9d rider (tightened by #567): this ``(COUNT(*), MAX(rowid))`` leg
|
|
1604
|
+
does NOT by itself catch the in-place ``ON CONFLICT(path) DO UPDATE SET
|
|
1363
1605
|
project_path = COALESCE(...)`` attribution backfill — that UPDATE preserves
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
attribution move never both slips this signal and leaves the cache stale.
|
|
1606
|
+
both values. `_ensure_session_files_row` therefore stamps the joined
|
|
1607
|
+
`session_entries` rows with a fresh mutation sequence when it fills either
|
|
1608
|
+
identity column. The envelope's mutation-sequence reconcile catches that
|
|
1609
|
+
pure metadata move even when no new entry is ingested.
|
|
1369
1610
|
"""
|
|
1370
1611
|
try:
|
|
1371
1612
|
row = cache_conn.execute(
|
|
@@ -5,7 +5,7 @@ import datetime as dt
|
|
|
5
5
|
import hashlib
|
|
6
6
|
import math
|
|
7
7
|
from collections import defaultdict
|
|
8
|
-
from dataclasses import dataclass, replace
|
|
8
|
+
from dataclasses import dataclass, field, replace
|
|
9
9
|
from typing import Generic, Iterable, Literal, TypeVar
|
|
10
10
|
|
|
11
11
|
|
|
@@ -79,6 +79,11 @@ class QualifiedCodexEntry:
|
|
|
79
79
|
# contributes to. The default is the reserved `unattributed` sentinel, so
|
|
80
80
|
# every existing constructor (and the whole CLI path) is unchanged.
|
|
81
81
|
account_key: str = "unattributed"
|
|
82
|
+
# Internal stable ordering identity from cache.db. It is deliberately
|
|
83
|
+
# excluded from equality and representation: provider outputs do not carry
|
|
84
|
+
# it, while #582's dirty-path replacement needs the original SQL tie-break
|
|
85
|
+
# when rows share timestamp/root/conversation.
|
|
86
|
+
cache_entry_id: int = field(default=0, compare=False, repr=False)
|
|
82
87
|
|
|
83
88
|
|
|
84
89
|
def emitted_project_label(entry: QualifiedCodexEntry) -> str:
|
|
@@ -205,7 +210,20 @@ class TokenTotals:
|
|
|
205
210
|
|
|
206
211
|
|
|
207
212
|
def _totals(entries: Iterable[QualifiedCodexEntry]) -> TokenTotals:
|
|
208
|
-
|
|
213
|
+
# #566 §5.1 item 7. `TokenTotals` carries no label, and nothing this
|
|
214
|
+
# function returns can observe one, so the collision-safe allocation this
|
|
215
|
+
# line used to run was dead work: it re-created every entry through
|
|
216
|
+
# `dataclasses.replace` purely to sum six numeric fields off the copies.
|
|
217
|
+
# On the maintainer's store the 148 calls from here drove 775,568
|
|
218
|
+
# `dataclasses.replace` calls and about 1.9s of every build.
|
|
219
|
+
#
|
|
220
|
+
# The allocation that MATTERS still runs, once, in
|
|
221
|
+
# `build_codex_project_result`, over the complete population, before any
|
|
222
|
+
# subset is taken -- which is the only place it can be correct. Guarding
|
|
223
|
+
# this call on "the subset holds more than one project identity" would
|
|
224
|
+
# have kept nearly all of the cost, because the expensive callers are the
|
|
225
|
+
# per-block totals over a population spanning every project.
|
|
226
|
+
values = tuple(entries)
|
|
209
227
|
return TokenTotals(
|
|
210
228
|
input_tokens=sum(entry.input_tokens for entry in values),
|
|
211
229
|
cached_input_tokens=sum(entry.cached_input_tokens for entry in values),
|