cctally 1.118.1 → 1.118.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/bin/_cctally_cache.py +465 -7
- package/bin/_cctally_dashboard.py +73 -7
- package/bin/_cctally_dashboard_sources.py +7 -0
- package/bin/_cctally_db.py +304 -0
- package/bin/_cctally_source_analytics.py +14 -3
- package/bin/_cctally_tui.py +189 -1
- package/bin/_lib_quota.py +31 -0
- package/bin/_lib_snapshot_cache.py +480 -1
- package/package.json +1 -1
|
@@ -44,6 +44,7 @@ reason.
|
|
|
44
44
|
"""
|
|
45
45
|
from __future__ import annotations
|
|
46
46
|
|
|
47
|
+
import collections
|
|
47
48
|
import contextlib
|
|
48
49
|
import datetime as dt
|
|
49
50
|
import hashlib
|
|
@@ -1404,8 +1405,12 @@ def reset_codex_accounting_cache_state() -> None:
|
|
|
1404
1405
|
#857: the consumed provenance token and its allocator are untouched. A
|
|
1405
1406
|
reset consumes nothing, and the next capture goes cold and mints a token
|
|
1406
1407
|
no earlier population carried.
|
|
1408
|
+
|
|
1409
|
+
#1059: a reset that discards pending forced paths collapses them into the
|
|
1410
|
+
requalify-all marker first, so the next capture re-qualifies everything.
|
|
1407
1411
|
"""
|
|
1408
1412
|
_assert_owner()
|
|
1413
|
+
_collapse_codex_forced_paths()
|
|
1409
1414
|
_CODEX_ACCOUNTING_CACHE_STATE.clear()
|
|
1410
1415
|
|
|
1411
1416
|
|
|
@@ -1434,6 +1439,60 @@ def restore_codex_accounting_cache_state(state: dict[str, object]) -> None:
|
|
|
1434
1439
|
_CODEX_ACCOUNTING_CACHE_STATE.update(state)
|
|
1435
1440
|
|
|
1436
1441
|
|
|
1442
|
+
# #1059 §6.4: the state key of the pending forced re-qualification set — an
|
|
1443
|
+
# immutable frozenset of `(source_root_key, source_path)` identities whose
|
|
1444
|
+
# retained `project_key` a filesystem regroup made stale. Inside the state
|
|
1445
|
+
# dictionary on purpose: a checkpoint copies it and a restore puts it back with
|
|
1446
|
+
# exactly the population it names.
|
|
1447
|
+
_CODEX_ACCOUNTING_FORCED_PATHS_KEY = "forced_paths"
|
|
1448
|
+
|
|
1449
|
+
|
|
1450
|
+
def _collapse_codex_forced_paths() -> None:
|
|
1451
|
+
"""Record that a population with pending forced paths is being discarded.
|
|
1452
|
+
|
|
1453
|
+
#1059 §6.4 (P1): the discarded paths cannot be named any more, so they
|
|
1454
|
+
collapse into `codex_requalify_all`, which lives in the identity state that
|
|
1455
|
+
no clear helper touches. The next capture goes cold and clears it.
|
|
1456
|
+
"""
|
|
1457
|
+
if _CODEX_ACCOUNTING_CACHE_STATE.get(_CODEX_ACCOUNTING_FORCED_PATHS_KEY):
|
|
1458
|
+
_PROJECT_IDENTITY_STATE["codex_requalify_all"] = True
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def force_codex_accounting_paths(select: "Callable[[object], bool]") -> int:
|
|
1462
|
+
"""Add the retained entries ``select`` accepts to the pending forced set.
|
|
1463
|
+
|
|
1464
|
+
#1059 §6.2 step 4: one scan of the retained population. Each selected
|
|
1465
|
+
entry contributes `(str(source_root_key), str(source_path))`; the rest of
|
|
1466
|
+
the state is left exactly as it is, so the forced paths are reloaded by the
|
|
1467
|
+
next warm capture's ordinary replacement path. Returns the number of
|
|
1468
|
+
identities newly added.
|
|
1469
|
+
"""
|
|
1470
|
+
_assert_owner()
|
|
1471
|
+
state = _CODEX_ACCOUNTING_CACHE_STATE
|
|
1472
|
+
if not state:
|
|
1473
|
+
# An empty state is cold by construction: nothing is retained, so
|
|
1474
|
+
# nothing can be stale.
|
|
1475
|
+
return 0
|
|
1476
|
+
pending = frozenset(state.get(_CODEX_ACCOUNTING_FORCED_PATHS_KEY) or ())
|
|
1477
|
+
selected = {
|
|
1478
|
+
(str(entry.source_root_key), str(entry.source_path))
|
|
1479
|
+
for entry in state.get("entries", ())
|
|
1480
|
+
if select(entry)
|
|
1481
|
+
}
|
|
1482
|
+
added = selected - pending
|
|
1483
|
+
if added:
|
|
1484
|
+
state[_CODEX_ACCOUNTING_FORCED_PATHS_KEY] = frozenset(pending | added)
|
|
1485
|
+
return len(added)
|
|
1486
|
+
|
|
1487
|
+
|
|
1488
|
+
def codex_accounting_work_outstanding() -> bool:
|
|
1489
|
+
"""Whether a regroup's Codex re-qualification has not been consumed yet."""
|
|
1490
|
+
return bool(
|
|
1491
|
+
_CODEX_ACCOUNTING_CACHE_STATE.get(_CODEX_ACCOUNTING_FORCED_PATHS_KEY)
|
|
1492
|
+
or _PROJECT_IDENTITY_STATE.get("codex_requalify_all")
|
|
1493
|
+
)
|
|
1494
|
+
|
|
1495
|
+
|
|
1437
1496
|
def _codex_accounting_mutation_seq(conn: sqlite3.Connection) -> int | None:
|
|
1438
1497
|
"""Return the dedicated ledger sequence, or ``None`` when unsupported."""
|
|
1439
1498
|
try:
|
|
@@ -1527,8 +1586,15 @@ def current_codex_population_signature() -> tuple | None:
|
|
|
1527
1586
|
|
|
1528
1587
|
|
|
1529
1588
|
def codex_accounting_cache_pending(conn: sqlite3.Connection) -> bool:
|
|
1530
|
-
"""Whether the process cache has not consumed the durable ledger head.
|
|
1589
|
+
"""Whether the process cache has not consumed the durable ledger head.
|
|
1590
|
+
|
|
1591
|
+
#1059 §6.4: also True while a regroup's Codex re-qualification is
|
|
1592
|
+
outstanding, so exact-version reuse and settled retention are refused
|
|
1593
|
+
until a capture consumes it.
|
|
1594
|
+
"""
|
|
1531
1595
|
_assert_owner()
|
|
1596
|
+
if codex_accounting_work_outstanding():
|
|
1597
|
+
return True
|
|
1532
1598
|
current = _codex_accounting_mutation_seq(conn)
|
|
1533
1599
|
if current is None or not _CODEX_ACCOUNTING_CACHE_STATE:
|
|
1534
1600
|
return True
|
|
@@ -1614,6 +1680,8 @@ def build_cached_codex_accounting(
|
|
|
1614
1680
|
cold = (
|
|
1615
1681
|
current_seq is None
|
|
1616
1682
|
or not state
|
|
1683
|
+
# #1059 §6.4: a discarded population had pending forced paths.
|
|
1684
|
+
or bool(_PROJECT_IDENTITY_STATE.get("codex_requalify_all"))
|
|
1617
1685
|
or state.get("extra") != extra_signature
|
|
1618
1686
|
or state.get("start") != range_start
|
|
1619
1687
|
or range_end < state.get("end", range_end)
|
|
@@ -1661,6 +1729,10 @@ def build_cached_codex_accounting(
|
|
|
1661
1729
|
)
|
|
1662
1730
|
except sqlite3.Error:
|
|
1663
1731
|
cold = True
|
|
1732
|
+
# #1059 §6.4: a regroup's forced re-qualification joins the dirty set
|
|
1733
|
+
# BEFORE the bound below, so forced plus ledger paths count together.
|
|
1734
|
+
if not cold:
|
|
1735
|
+
dirty.update(state.get(_CODEX_ACCOUNTING_FORCED_PATHS_KEY) or ())
|
|
1664
1736
|
# The qualified path loader emits two SQL parameters and one OR arm per
|
|
1665
1737
|
# identity. Beyond this bound a cold indexed range read is both safer
|
|
1666
1738
|
# and cheaper than approaching SQLite's variable/expression limits.
|
|
@@ -1668,6 +1740,7 @@ def build_cached_codex_accounting(
|
|
|
1668
1740
|
cold = True
|
|
1669
1741
|
|
|
1670
1742
|
if cold:
|
|
1743
|
+
note_project_work("codexLoadAll")
|
|
1671
1744
|
entries = tuple(sorted(tuple(load_all()), key=order_key))
|
|
1672
1745
|
if finalize_entries is not None:
|
|
1673
1746
|
entries = tuple(finalize_entries(entries, None))
|
|
@@ -1690,6 +1763,9 @@ def build_cached_codex_accounting(
|
|
|
1690
1763
|
"population_signature": population_signature,
|
|
1691
1764
|
_CODEX_ACCOUNTING_PROVENANCE_STATE_KEY: provenance_token,
|
|
1692
1765
|
})
|
|
1766
|
+
# #1059 §6.4: a committed cold capture re-qualified everything, so it
|
|
1767
|
+
# consumed every forced path (the clear above) and the marker.
|
|
1768
|
+
_PROJECT_IDENTITY_STATE["codex_requalify_all"] = False
|
|
1693
1769
|
return CodexAccountingCacheResult(
|
|
1694
1770
|
entries, (), accounts, True, prior_cached_entries, entries,
|
|
1695
1771
|
population_signature=population_signature,
|
|
@@ -1726,6 +1802,8 @@ def build_cached_codex_accounting(
|
|
|
1726
1802
|
prior_dirty = tuple(
|
|
1727
1803
|
entry for entry in prior_entries if path_of(entry) in dirty
|
|
1728
1804
|
)
|
|
1805
|
+
note_project_work("codexLoadPaths")
|
|
1806
|
+
note_project_work("codexLoadPathsPaths", len(dirty_paths))
|
|
1729
1807
|
replacements = tuple(load_paths(dirty_paths))
|
|
1730
1808
|
entries = tuple(sorted(
|
|
1731
1809
|
(
|
|
@@ -1769,6 +1847,9 @@ def build_cached_codex_accounting(
|
|
|
1769
1847
|
state["generation"] = generation
|
|
1770
1848
|
state["population_signature"] = population_signature
|
|
1771
1849
|
state[_CODEX_ACCOUNTING_PROVENANCE_STATE_KEY] = provenance_token
|
|
1850
|
+
# #1059 §6.4: the forced paths were reloaded as part of `dirty`, so the
|
|
1851
|
+
# committed state no longer carries them.
|
|
1852
|
+
state.pop(_CODEX_ACCOUNTING_FORCED_PATHS_KEY, None)
|
|
1772
1853
|
return CodexAccountingCacheResult(
|
|
1773
1854
|
entries, dirty_paths, dirty_accounts, False, changed_old, changed_new,
|
|
1774
1855
|
population_signature=population_signature,
|
|
@@ -2009,6 +2090,51 @@ def reset_dispatch_state() -> None:
|
|
|
2009
2090
|
_LAST_PUBLISHED_SNAPSHOT = None
|
|
2010
2091
|
|
|
2011
2092
|
|
|
2093
|
+
# === #1059 SD3 — project work counters =======================================
|
|
2094
|
+
#
|
|
2095
|
+
# Cumulative counts of the expensive work sites a warm tick can reach, so the
|
|
2096
|
+
# claim "an unchanged filesystem causes no closed-week rebuild" is measured at
|
|
2097
|
+
# the work itself rather than inferred from a reconcile flag (`hit=True` on the
|
|
2098
|
+
# projects reconcile means only that reconciliation succeeded). Plain integers,
|
|
2099
|
+
# incremented only and never reset in production; `/api/debug/backend` reads a
|
|
2100
|
+
# copy. Deliberately not an accelerator root: they hold no cached value, so
|
|
2101
|
+
# memory enforcement must never clear them.
|
|
2102
|
+
_PROJECT_WORK_KINDS = (
|
|
2103
|
+
"closedWeekComputes",
|
|
2104
|
+
"fullWindowProjectWalks",
|
|
2105
|
+
"currentWeekColdFolds",
|
|
2106
|
+
"weekrefCostComputes",
|
|
2107
|
+
"rangeFullRefolds",
|
|
2108
|
+
"codexLoadAll",
|
|
2109
|
+
"codexLoadPaths",
|
|
2110
|
+
"codexLoadPathsPaths",
|
|
2111
|
+
)
|
|
2112
|
+
_PROJECT_WORK_LOCK = threading.Lock()
|
|
2113
|
+
_PROJECT_WORK_COUNTS: dict[str, int] = dict.fromkeys(_PROJECT_WORK_KINDS, 0)
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def note_project_work(kind: str, n: int = 1) -> None:
|
|
2117
|
+
"""Add ``n`` to one work counter; an unknown ``kind`` raises ``ValueError``."""
|
|
2118
|
+
if kind not in _PROJECT_WORK_COUNTS:
|
|
2119
|
+
raise ValueError(f"unknown project work kind: {kind!r}")
|
|
2120
|
+
with _PROJECT_WORK_LOCK:
|
|
2121
|
+
_PROJECT_WORK_COUNTS[kind] += int(n)
|
|
2122
|
+
|
|
2123
|
+
|
|
2124
|
+
def project_work_stats() -> dict[str, int]:
|
|
2125
|
+
"""A copy of every work counter (all eight keys, integers)."""
|
|
2126
|
+
with _PROJECT_WORK_LOCK:
|
|
2127
|
+
return {kind: int(_PROJECT_WORK_COUNTS[kind])
|
|
2128
|
+
for kind in _PROJECT_WORK_KINDS}
|
|
2129
|
+
|
|
2130
|
+
|
|
2131
|
+
def reset_project_work_stats() -> None:
|
|
2132
|
+
"""Zero every work counter (test hook only)."""
|
|
2133
|
+
with _PROJECT_WORK_LOCK:
|
|
2134
|
+
for kind in _PROJECT_WORK_KINDS:
|
|
2135
|
+
_PROJECT_WORK_COUNTS[kind] = 0
|
|
2136
|
+
|
|
2137
|
+
|
|
2012
2138
|
# === #269 M0 — shared per-weekref immutable-cost cache (B1 trend + B3 forecast)
|
|
2013
2139
|
#
|
|
2014
2140
|
# A closed subscription week's cost is IMMUTABLE, so it is computed once and
|
|
@@ -2067,11 +2193,13 @@ def cached_weekref_cost(*, week_start_at, week_end_at, now_utc, compute):
|
|
|
2067
2193
|
the returned float is bit-identical to today's.
|
|
2068
2194
|
"""
|
|
2069
2195
|
if week_end_at > now_utc:
|
|
2196
|
+
note_project_work("weekrefCostComputes")
|
|
2070
2197
|
return compute()
|
|
2071
2198
|
key = _weekref_key(week_start_at, week_end_at)
|
|
2072
2199
|
hit = _WEEKREF_COST_CACHE.get(key)
|
|
2073
2200
|
if hit is not None: # 0.0 is a legitimate cached value, not a miss
|
|
2074
2201
|
return hit
|
|
2202
|
+
note_project_work("weekrefCostComputes")
|
|
2075
2203
|
val = compute()
|
|
2076
2204
|
_WEEKREF_COST_CACHE[key] = val
|
|
2077
2205
|
return val
|
|
@@ -2424,6 +2552,7 @@ def accumulate_projects_current_week(*, week_key, cur_max_id, cur_max_seq,
|
|
|
2424
2552
|
):
|
|
2425
2553
|
cold = True
|
|
2426
2554
|
if cold:
|
|
2555
|
+
note_project_work("currentWeekColdFolds")
|
|
2427
2556
|
mut, week_total, tail = fetch_all_raw()
|
|
2428
2557
|
_PROJECTS_ENV_CURRENT["state"] = {
|
|
2429
2558
|
"label": week_key,
|
|
@@ -2545,6 +2674,353 @@ def reconcile_projects_env_cache(cache_conn, *, max_entry_id, max_mutation_seq,
|
|
|
2545
2674
|
ls["sf_sig"] = sf_sig
|
|
2546
2675
|
|
|
2547
2676
|
|
|
2677
|
+
def evict_projects_env_weeks_for_buckets(buckets) -> int:
|
|
2678
|
+
"""Evict every cached CLOSED week holding a bucket in ``buckets`` (#1059).
|
|
2679
|
+
|
|
2680
|
+
A week is evicted whole: all of its ``(bucket_path, week_iso)`` rows AND
|
|
2681
|
+
its ``_PROJECTS_ENV_WEEK_TOTALS`` entry. The totals dictionary is the hit
|
|
2682
|
+
registry (``projects_env_week_get``), so deleting the affected bucket rows
|
|
2683
|
+
alone would leave a HIT that silently lacks those rows. Weeks that hold no
|
|
2684
|
+
affected bucket, and ``_PROJECTS_ENV_LAST_SEEN``, are untouched, so an
|
|
2685
|
+
unaffected week stays a hit. Returns the number of weeks evicted.
|
|
2686
|
+
"""
|
|
2687
|
+
_assert_owner()
|
|
2688
|
+
affected = frozenset(buckets)
|
|
2689
|
+
weeks = {wk for (bp, wk) in _PROJECTS_ENV_WEEK_CACHE if bp in affected}
|
|
2690
|
+
for wk in weeks:
|
|
2691
|
+
_PROJECTS_ENV_WEEK_TOTALS.pop(wk, None)
|
|
2692
|
+
for key in [k for k in _PROJECTS_ENV_WEEK_CACHE if k[1] == wk]:
|
|
2693
|
+
del _PROJECTS_ENV_WEEK_CACHE[key]
|
|
2694
|
+
return len(weeks)
|
|
2695
|
+
|
|
2696
|
+
|
|
2697
|
+
def reset_projects_env_current_if_holds(buckets) -> bool:
|
|
2698
|
+
"""Reset the CURRENT-week slot iff its retained ``mut`` has a key in
|
|
2699
|
+
``buckets`` (#1059); return whether it did. The slot's ``mut`` is keyed by
|
|
2700
|
+
``bucket_path`` (the dashboard's ``_fold_projects_entry`` shape), which is
|
|
2701
|
+
the one fact this module reads from it."""
|
|
2702
|
+
_assert_owner()
|
|
2703
|
+
state = _PROJECTS_ENV_CURRENT["state"]
|
|
2704
|
+
if not isinstance(state, dict):
|
|
2705
|
+
return False
|
|
2706
|
+
mut = state.get("mut")
|
|
2707
|
+
if not isinstance(mut, dict) or not any(bp in mut for bp in buckets):
|
|
2708
|
+
return False
|
|
2709
|
+
reset_projects_env_current_state()
|
|
2710
|
+
return True
|
|
2711
|
+
|
|
2712
|
+
|
|
2713
|
+
# === #1059 — warm project regrouping: observation registry + cadence sweep ===
|
|
2714
|
+
#
|
|
2715
|
+
# Every warm consumer keeps its own resolver memo with today's semantics (so
|
|
2716
|
+
# labels and `first_key` values never change on an unchanged filesystem). In
|
|
2717
|
+
# owner builds those memos are `ObservedProjectMemo`s, which report each raw
|
|
2718
|
+
# spelling's resolutions to the owner-held registry below. The registry keeps
|
|
2719
|
+
# the SET of identities observed per spelling since the last successful sweep:
|
|
2720
|
+
# a consumer can resolve a spelling under an intermediate identity between two
|
|
2721
|
+
# sweeps and retain the result, and the filesystem can then move on or return,
|
|
2722
|
+
# so comparing only the first or the latest observation would miss retained
|
|
2723
|
+
# state. A cadence-gated sweep re-resolves every registered spelling with one
|
|
2724
|
+
# fresh plain memo; a spelling flips when its set holds any identity other than
|
|
2725
|
+
# the fresh one. The orchestration (`bin/_cctally_tui.py::
|
|
2726
|
+
# _tui_revalidate_project_identities`) invalidates exactly the state built from
|
|
2727
|
+
# the stale buckets and bumps the generation once.
|
|
2728
|
+
#
|
|
2729
|
+
# `_PROJECT_IDENTITY_STATE` is deliberately a PLAIN dict, not an
|
|
2730
|
+
# `_ObservedSnapshotDict`, and is never one of `_snapshot_accelerator_roots`:
|
|
2731
|
+
# filesystem identity does not depend on database contents, so neither memory
|
|
2732
|
+
# enforcement, nor the orphan-prune site, nor a cold or accounting reset may
|
|
2733
|
+
# clear it — observation coverage never shrinks. Its size is O(distinct
|
|
2734
|
+
# spellings the process resolves in owner builds) and it is reported
|
|
2735
|
+
# separately (`project_identity_stats`), never under the accelerator budget.
|
|
2736
|
+
# The resolver and the clock are injected, keeping this module free of
|
|
2737
|
+
# dashboard imports.
|
|
2738
|
+
|
|
2739
|
+
#: The revalidation interval (operator decision Q-OP1): a change is detected
|
|
2740
|
+
#: within about 70 s at the default 10 s sync interval.
|
|
2741
|
+
PROJECT_IDENTITY_SWEEP_INTERVAL_S: float = 60.0
|
|
2742
|
+
_PROJECT_IDENTITY_RING_CAPACITY = 64
|
|
2743
|
+
_PROJECT_IDENTITY_FAILURES = (OSError, RuntimeError, ValueError)
|
|
2744
|
+
|
|
2745
|
+
_PROJECT_IDENTITY_STATE: dict = {
|
|
2746
|
+
"observed": {},
|
|
2747
|
+
"next_due": None,
|
|
2748
|
+
"regroup_generation": None,
|
|
2749
|
+
"codex_requalify_all": False,
|
|
2750
|
+
}
|
|
2751
|
+
_PROJECT_IDENTITY_STATS_LOCK = threading.Lock()
|
|
2752
|
+
|
|
2753
|
+
|
|
2754
|
+
def _new_project_identity_stats() -> dict:
|
|
2755
|
+
return {
|
|
2756
|
+
"sweepAttempts": 0,
|
|
2757
|
+
"sweepSuccesses": 0,
|
|
2758
|
+
"sweepFailures": 0,
|
|
2759
|
+
"spellingsVisited": 0,
|
|
2760
|
+
"flips": 0,
|
|
2761
|
+
"flipBatches": 0,
|
|
2762
|
+
"retryBumps": 0,
|
|
2763
|
+
"lastSweepMs": 0.0,
|
|
2764
|
+
"totalSweepMs": 0.0,
|
|
2765
|
+
}
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
_PROJECT_IDENTITY_STATS: dict = _new_project_identity_stats()
|
|
2769
|
+
_PROJECT_IDENTITY_RECENT: "collections.deque[dict]" = collections.deque(
|
|
2770
|
+
maxlen=_PROJECT_IDENTITY_RING_CAPACITY)
|
|
2771
|
+
#: Never rewound, not even by the test reset: a sweep record's `id` names one
|
|
2772
|
+
#: attempt for the life of the process.
|
|
2773
|
+
_PROJECT_IDENTITY_SWEEP_IDS = itertools.count(1)
|
|
2774
|
+
|
|
2775
|
+
|
|
2776
|
+
def project_identity_fields(key) -> tuple:
|
|
2777
|
+
"""The compared identity of a resolved ``ProjectKey``.
|
|
2778
|
+
|
|
2779
|
+
``display_key`` is excluded: it depends on the order in which one memo saw
|
|
2780
|
+
aliases of a directory and carries no filesystem information beyond
|
|
2781
|
+
``bucket_path`` and ``git_root``. ``ProjectKey`` equality is never used,
|
|
2782
|
+
because it compares ``bucket_path`` only and would miss a same-bucket
|
|
2783
|
+
metadata change (a ``git init`` in a no-git directory).
|
|
2784
|
+
"""
|
|
2785
|
+
return (
|
|
2786
|
+
key.bucket_path,
|
|
2787
|
+
key.git_root,
|
|
2788
|
+
bool(key.is_unknown),
|
|
2789
|
+
bool(key.is_no_git),
|
|
2790
|
+
)
|
|
2791
|
+
|
|
2792
|
+
|
|
2793
|
+
def _record_project_observation(spelling: str, key) -> None:
|
|
2794
|
+
_assert_owner()
|
|
2795
|
+
fields = project_identity_fields(key)
|
|
2796
|
+
observed = _PROJECT_IDENTITY_STATE["observed"]
|
|
2797
|
+
seen = observed.get(spelling)
|
|
2798
|
+
if seen is None:
|
|
2799
|
+
observed[spelling] = {fields}
|
|
2800
|
+
else:
|
|
2801
|
+
seen.add(fields)
|
|
2802
|
+
|
|
2803
|
+
|
|
2804
|
+
class ObservedProjectMemo(dict):
|
|
2805
|
+
"""A resolver memo that reports every raw-spelling resolution (#1059).
|
|
2806
|
+
|
|
2807
|
+
``_resolve_project_key`` writes its memo only by subscript assignment, so
|
|
2808
|
+
overriding ``__setitem__`` sees every write without a resolver change.
|
|
2809
|
+
Only ``("raw", spelling)`` keys are recorded; the normalized string keys
|
|
2810
|
+
are the resolver's internal entries. Reads behave exactly like a plain
|
|
2811
|
+
dict, so a consumer's labels are the ones its own memo would produce.
|
|
2812
|
+
"""
|
|
2813
|
+
|
|
2814
|
+
def __setitem__(self, key, value):
|
|
2815
|
+
super().__setitem__(key, value)
|
|
2816
|
+
if (
|
|
2817
|
+
isinstance(key, tuple)
|
|
2818
|
+
and len(key) == 2
|
|
2819
|
+
and key[0] == "raw"
|
|
2820
|
+
and isinstance(key[1], str)
|
|
2821
|
+
):
|
|
2822
|
+
_record_project_observation(key[1], value)
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
def observing_project_memo(initial=None) -> ObservedProjectMemo:
|
|
2826
|
+
"""A new observing memo, copied from ``initial`` when given.
|
|
2827
|
+
|
|
2828
|
+
Owner-asserted: only an owner build may feed the registry. Copying never
|
|
2829
|
+
calls ``__setitem__``, and every copied raw entry was recorded when it was
|
|
2830
|
+
first written by the memo it came from.
|
|
2831
|
+
"""
|
|
2832
|
+
_assert_owner()
|
|
2833
|
+
if initial is None:
|
|
2834
|
+
return ObservedProjectMemo()
|
|
2835
|
+
return ObservedProjectMemo(initial)
|
|
2836
|
+
|
|
2837
|
+
|
|
2838
|
+
@dataclass(frozen=True)
|
|
2839
|
+
class ProjectIdentityFlip:
|
|
2840
|
+
"""One spelling whose observed identities differ from its fresh one."""
|
|
2841
|
+
|
|
2842
|
+
spelling: str
|
|
2843
|
+
stale: frozenset
|
|
2844
|
+
fresh: tuple
|
|
2845
|
+
|
|
2846
|
+
|
|
2847
|
+
@dataclass(frozen=True)
|
|
2848
|
+
class ProjectIdentitySweep:
|
|
2849
|
+
"""The result of one successful due sweep (possibly with no flips)."""
|
|
2850
|
+
|
|
2851
|
+
flips: tuple
|
|
2852
|
+
|
|
2853
|
+
@property
|
|
2854
|
+
def old_buckets(self) -> frozenset:
|
|
2855
|
+
"""The ``bucket_path`` of every stale identity of every flip."""
|
|
2856
|
+
return frozenset(
|
|
2857
|
+
stale[0] for flip in self.flips for stale in flip.stale)
|
|
2858
|
+
|
|
2859
|
+
|
|
2860
|
+
def _note_project_identity_sweep(*, ms: float, spellings: int, flips: int,
|
|
2861
|
+
ok: bool) -> None:
|
|
2862
|
+
with _PROJECT_IDENTITY_STATS_LOCK:
|
|
2863
|
+
stats = _PROJECT_IDENTITY_STATS
|
|
2864
|
+
stats["sweepAttempts"] += 1
|
|
2865
|
+
stats["sweepSuccesses" if ok else "sweepFailures"] += 1
|
|
2866
|
+
stats["spellingsVisited"] += int(spellings)
|
|
2867
|
+
stats["flips"] += int(flips)
|
|
2868
|
+
if ok and flips:
|
|
2869
|
+
stats["flipBatches"] += 1
|
|
2870
|
+
stats["lastSweepMs"] = float(ms)
|
|
2871
|
+
stats["totalSweepMs"] += float(ms)
|
|
2872
|
+
_PROJECT_IDENTITY_RECENT.append({
|
|
2873
|
+
"id": next(_PROJECT_IDENTITY_SWEEP_IDS),
|
|
2874
|
+
"at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
2875
|
+
"ms": float(ms),
|
|
2876
|
+
"spellings": int(spellings),
|
|
2877
|
+
"flips": int(flips),
|
|
2878
|
+
"outcome": "ok" if ok else "failed",
|
|
2879
|
+
"retryBump": False,
|
|
2880
|
+
})
|
|
2881
|
+
|
|
2882
|
+
|
|
2883
|
+
def revalidate_project_identities(
|
|
2884
|
+
*,
|
|
2885
|
+
resolve: "Callable[[str, str, dict], object]",
|
|
2886
|
+
now_monotonic: float,
|
|
2887
|
+
interval: float = PROJECT_IDENTITY_SWEEP_INTERVAL_S,
|
|
2888
|
+
) -> "ProjectIdentitySweep | None":
|
|
2889
|
+
"""Run the cadence-gated identity sweep when it is due.
|
|
2890
|
+
|
|
2891
|
+
Not due (or the very first call, which only arms the deadline): ``None``.
|
|
2892
|
+
Due: every registered spelling is re-resolved through
|
|
2893
|
+
``resolve(spelling, "git-root", fresh)`` with ONE fresh plain dictionary
|
|
2894
|
+
(alias sharing inside it cannot change a compared field, because all of
|
|
2895
|
+
them derive from the normalized path). On success each spelling's set is
|
|
2896
|
+
replaced by ``{fresh}``, the deadline moves to ``now + interval``, and the
|
|
2897
|
+
sweep is returned with every spelling whose set held another identity. A
|
|
2898
|
+
failed resolution (``OSError`` / ``RuntimeError`` / ``ValueError``) keeps
|
|
2899
|
+
the registry AND the deadline, so the next build retries, and returns
|
|
2900
|
+
``None``.
|
|
2901
|
+
"""
|
|
2902
|
+
_assert_owner()
|
|
2903
|
+
state = _PROJECT_IDENTITY_STATE
|
|
2904
|
+
next_due = state["next_due"]
|
|
2905
|
+
if next_due is None:
|
|
2906
|
+
state["next_due"] = now_monotonic + interval
|
|
2907
|
+
return None
|
|
2908
|
+
if now_monotonic < next_due:
|
|
2909
|
+
return None
|
|
2910
|
+
observed = state["observed"]
|
|
2911
|
+
spellings = sorted(observed)
|
|
2912
|
+
fresh_memo: dict = {}
|
|
2913
|
+
fresh: dict = {}
|
|
2914
|
+
started = time.perf_counter()
|
|
2915
|
+
try:
|
|
2916
|
+
for spelling in spellings:
|
|
2917
|
+
fresh[spelling] = project_identity_fields(
|
|
2918
|
+
resolve(spelling, "git-root", fresh_memo))
|
|
2919
|
+
except _PROJECT_IDENTITY_FAILURES:
|
|
2920
|
+
_note_project_identity_sweep(
|
|
2921
|
+
ms=(time.perf_counter() - started) * 1000.0,
|
|
2922
|
+
spellings=len(fresh), flips=0, ok=False)
|
|
2923
|
+
return None
|
|
2924
|
+
flips = []
|
|
2925
|
+
for spelling in spellings:
|
|
2926
|
+
now_fields = fresh[spelling]
|
|
2927
|
+
stale = frozenset(
|
|
2928
|
+
fields for fields in observed[spelling] if fields != now_fields)
|
|
2929
|
+
if stale:
|
|
2930
|
+
flips.append(ProjectIdentityFlip(spelling, stale, now_fields))
|
|
2931
|
+
state["observed"] = {
|
|
2932
|
+
spelling: {fields} for spelling, fields in fresh.items()}
|
|
2933
|
+
state["next_due"] = now_monotonic + interval
|
|
2934
|
+
_note_project_identity_sweep(
|
|
2935
|
+
ms=(time.perf_counter() - started) * 1000.0,
|
|
2936
|
+
spellings=len(spellings), flips=len(flips), ok=True)
|
|
2937
|
+
return ProjectIdentitySweep(tuple(flips))
|
|
2938
|
+
|
|
2939
|
+
|
|
2940
|
+
def restore_project_identity_flips(sweep: ProjectIdentitySweep) -> None:
|
|
2941
|
+
"""Put a sweep's stale identities back after its flip steps failed.
|
|
2942
|
+
|
|
2943
|
+
A successful sweep replaces every set with ``{fresh}`` before the caller
|
|
2944
|
+
applies the flips; if one of those steps raises, nothing else remembers
|
|
2945
|
+
the stale identity and the next sweep would see no flip. Unioning each
|
|
2946
|
+
flip's ``stale`` set back into its spelling's set makes the next due
|
|
2947
|
+
sweep detect the same flip again. The deadline is untouched, so the retry
|
|
2948
|
+
happens at the next interval rather than on every build, and the sweep's
|
|
2949
|
+
counters and ring record keep describing the sweep as it ran.
|
|
2950
|
+
"""
|
|
2951
|
+
_assert_owner()
|
|
2952
|
+
observed = _PROJECT_IDENTITY_STATE["observed"]
|
|
2953
|
+
for flip in sweep.flips:
|
|
2954
|
+
observed.setdefault(flip.spelling, set()).update(flip.stale)
|
|
2955
|
+
|
|
2956
|
+
|
|
2957
|
+
def mark_project_regroup_pending(generation: int) -> None:
|
|
2958
|
+
"""Record ``generation`` as the pending regroup generation ``G``."""
|
|
2959
|
+
_assert_owner()
|
|
2960
|
+
_PROJECT_IDENTITY_STATE["regroup_generation"] = int(generation)
|
|
2961
|
+
|
|
2962
|
+
|
|
2963
|
+
def project_regroup_pending() -> "int | None":
|
|
2964
|
+
"""The pending regroup generation, or ``None`` when nothing is pending."""
|
|
2965
|
+
return _PROJECT_IDENTITY_STATE["regroup_generation"]
|
|
2966
|
+
|
|
2967
|
+
|
|
2968
|
+
def acknowledge_project_regroup() -> None:
|
|
2969
|
+
"""Clear the pending regroup generation once a build republished it."""
|
|
2970
|
+
_assert_owner()
|
|
2971
|
+
_PROJECT_IDENTITY_STATE["regroup_generation"] = None
|
|
2972
|
+
|
|
2973
|
+
|
|
2974
|
+
def note_project_identity_retry_bump() -> None:
|
|
2975
|
+
"""Count one cadence retry bump and mark it on the latest sweep record."""
|
|
2976
|
+
with _PROJECT_IDENTITY_STATS_LOCK:
|
|
2977
|
+
_PROJECT_IDENTITY_STATS["retryBumps"] += 1
|
|
2978
|
+
if _PROJECT_IDENTITY_RECENT:
|
|
2979
|
+
_PROJECT_IDENTITY_RECENT[-1]["retryBump"] = True
|
|
2980
|
+
|
|
2981
|
+
|
|
2982
|
+
def project_identity_stats() -> dict:
|
|
2983
|
+
"""The ``projectIdentity`` diagnostic block (spec section 7)."""
|
|
2984
|
+
with _PROJECT_IDENTITY_STATS_LOCK:
|
|
2985
|
+
stats = dict(_PROJECT_IDENTITY_STATS)
|
|
2986
|
+
recent = [dict(record) for record in _PROJECT_IDENTITY_RECENT]
|
|
2987
|
+
state = _PROJECT_IDENTITY_STATE
|
|
2988
|
+
regroup = state["regroup_generation"]
|
|
2989
|
+
forced = _CODEX_ACCOUNTING_CACHE_STATE.get("forced_paths") or ()
|
|
2990
|
+
return {
|
|
2991
|
+
"sweepAttempts": int(stats["sweepAttempts"]),
|
|
2992
|
+
"sweepSuccesses": int(stats["sweepSuccesses"]),
|
|
2993
|
+
"sweepFailures": int(stats["sweepFailures"]),
|
|
2994
|
+
"spellingsVisited": int(stats["spellingsVisited"]),
|
|
2995
|
+
"flips": int(stats["flips"]),
|
|
2996
|
+
"flipBatches": int(stats["flipBatches"]),
|
|
2997
|
+
"retryBumps": int(stats["retryBumps"]),
|
|
2998
|
+
"lastSweepMs": float(stats["lastSweepMs"]),
|
|
2999
|
+
"totalSweepMs": float(stats["totalSweepMs"]),
|
|
3000
|
+
"observedSpellings": len(state["observed"]),
|
|
3001
|
+
"regroupGeneration": None if regroup is None else int(regroup),
|
|
3002
|
+
"forcedCodexPathsPending": len(forced),
|
|
3003
|
+
"codexRequalifyAll": bool(state["codex_requalify_all"]),
|
|
3004
|
+
"recentSweeps": recent,
|
|
3005
|
+
}
|
|
3006
|
+
|
|
3007
|
+
|
|
3008
|
+
def reset_project_identity_state() -> None:
|
|
3009
|
+
"""Drop the registry, deadline, markers and counters (test hook only).
|
|
3010
|
+
|
|
3011
|
+
Not owner-asserted, so the per-test autouse reset works even after an
|
|
3012
|
+
earlier test armed the owner on a thread that has since exited.
|
|
3013
|
+
"""
|
|
3014
|
+
_PROJECT_IDENTITY_STATE["observed"] = {}
|
|
3015
|
+
_PROJECT_IDENTITY_STATE["next_due"] = None
|
|
3016
|
+
_PROJECT_IDENTITY_STATE["regroup_generation"] = None
|
|
3017
|
+
_PROJECT_IDENTITY_STATE["codex_requalify_all"] = False
|
|
3018
|
+
with _PROJECT_IDENTITY_STATS_LOCK:
|
|
3019
|
+
_PROJECT_IDENTITY_STATS.clear()
|
|
3020
|
+
_PROJECT_IDENTITY_STATS.update(_new_project_identity_stats())
|
|
3021
|
+
_PROJECT_IDENTITY_RECENT.clear()
|
|
3022
|
+
|
|
3023
|
+
|
|
2548
3024
|
# === #272 — cache-report per-day cache =====================================
|
|
2549
3025
|
#
|
|
2550
3026
|
# A per-day cache in front of the dashboard's `build_cache_report_snapshot`
|
|
@@ -2804,6 +3280,9 @@ def _snapshot_accelerator_entry_count() -> int:
|
|
|
2804
3280
|
|
|
2805
3281
|
def _clear_snapshot_accelerators() -> None:
|
|
2806
3282
|
"""Response-preserving overflow fallback: the next tick recomputes cold."""
|
|
3283
|
+
# #1059 §6.4: eviction discards the Codex population too; pending forced
|
|
3284
|
+
# paths survive it as the requalify-all marker.
|
|
3285
|
+
_collapse_codex_forced_paths()
|
|
2807
3286
|
for root in _snapshot_accelerator_roots():
|
|
2808
3287
|
root.clear()
|
|
2809
3288
|
# Preserve the single-slot mapping shape expected by its callers.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cctally",
|
|
3
|
-
"version": "1.118.
|
|
3
|
+
"version": "1.118.2",
|
|
4
4
|
"description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
|
|
5
5
|
"homepage": "https://github.com/omrikais/cctally",
|
|
6
6
|
"repository": {
|