cctally 1.112.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +4 -5
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +209 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_lib_rederive.py
CHANGED
|
@@ -194,7 +194,15 @@ def validate_family_registry(*, evt_kinds: set[str],
|
|
|
194
194
|
)
|
|
195
195
|
|
|
196
196
|
|
|
197
|
+
# #874: the contract is structural. It names every column any planner reader
|
|
198
|
+
# of the table reads (`_validate_cache_rows`, `_cache_fingerprint`,
|
|
199
|
+
# `_joined_entries`, `iter_entries`), not only those the current journal
|
|
200
|
+
# reaches, so whether a cache is accepted never depends on journal or pricing
|
|
201
|
+
# state, and a missing column is a typed gap rather than a raw sqlite error.
|
|
197
202
|
_SESSION_ENTRY_COLUMNS = frozenset({
|
|
203
|
+
"id",
|
|
204
|
+
"source_path",
|
|
205
|
+
"line_offset",
|
|
198
206
|
"timestamp_utc",
|
|
199
207
|
"model",
|
|
200
208
|
"input_tokens",
|
|
@@ -202,7 +210,8 @@ _SESSION_ENTRY_COLUMNS = frozenset({
|
|
|
202
210
|
"cache_create_tokens",
|
|
203
211
|
"cache_read_tokens",
|
|
204
212
|
"cache_create_1h_tokens",
|
|
205
|
-
"
|
|
213
|
+
"cost_usd_raw",
|
|
214
|
+
"speed",
|
|
206
215
|
"account_key",
|
|
207
216
|
})
|
|
208
217
|
_SESSION_FILE_COLUMNS = frozenset({"path", "session_id", "project_path"})
|
|
@@ -228,6 +237,37 @@ def validate_claude_cache_contract(tables: Mapping[str, set[str]]) -> None:
|
|
|
228
237
|
)
|
|
229
238
|
|
|
230
239
|
|
|
240
|
+
# #874: the Codex columns `_cache_fingerprint(include_codex=True)` and the
|
|
241
|
+
# Codex budget re-pricing reader (`iter_codex_entries`) read. Required only
|
|
242
|
+
# when the planner reads the Codex cache at all.
|
|
243
|
+
_CODEX_SESSION_ENTRY_COLUMNS = frozenset({
|
|
244
|
+
"source_path",
|
|
245
|
+
"line_offset",
|
|
246
|
+
"timestamp_utc",
|
|
247
|
+
"session_id",
|
|
248
|
+
"model",
|
|
249
|
+
"input_tokens",
|
|
250
|
+
"cached_input_tokens",
|
|
251
|
+
"output_tokens",
|
|
252
|
+
"reasoning_output_tokens",
|
|
253
|
+
"total_tokens",
|
|
254
|
+
"account_key",
|
|
255
|
+
})
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def validate_codex_cache_contract(tables: Mapping[str, set[str]]) -> None:
|
|
259
|
+
"""Fail before planning if the Codex cost inputs the planner reads are gone."""
|
|
260
|
+
if "codex_session_entries" not in tables:
|
|
261
|
+
raise RederiveDataGap("missing cache.db table codex_session_entries")
|
|
262
|
+
missing = sorted(
|
|
263
|
+
_CODEX_SESSION_ENTRY_COLUMNS - set(tables["codex_session_entries"]))
|
|
264
|
+
if missing:
|
|
265
|
+
raise RederiveDataGap(
|
|
266
|
+
"missing cache.db codex_session_entries column(s): "
|
|
267
|
+
+ ", ".join(missing)
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
|
|
231
271
|
def preserved_history(records: Iterable[Mapping], *,
|
|
232
272
|
evidence_retained: bool) -> dict[str, Mapping]:
|
|
233
273
|
"""Owned events the current re-derivation cannot reach, keyed by event id.
|
|
@@ -1272,7 +1312,7 @@ def _reconcile_snapshot_identities(
|
|
|
1272
1312
|
|
|
1273
1313
|
def _preserve_reviewed_hold_dependencies(
|
|
1274
1314
|
current: Mapping[str, Mapping], desired: dict[str, Mapping],
|
|
1275
|
-
reviewed_weekly_hold_ids=(),
|
|
1315
|
+
reviewed_weekly_hold_ids=(), historical_close_markers=None,
|
|
1276
1316
|
) -> None:
|
|
1277
1317
|
"""Keep frozen five-hour facts reached from exact reviewed holds.
|
|
1278
1318
|
|
|
@@ -1413,7 +1453,9 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1413
1453
|
|
|
1414
1454
|
held_windows = set()
|
|
1415
1455
|
for snapshot_id in snapshot_ids:
|
|
1416
|
-
|
|
1456
|
+
# #885: only final support may freeze a close. A current snapshot
|
|
1457
|
+
# absent from this dependency-closed target is retired by this plan.
|
|
1458
|
+
payload = (desired.get(snapshot_id) or {}).get("payload") or {}
|
|
1417
1459
|
if payload.get("kind") != "snapshot_accept":
|
|
1418
1460
|
continue
|
|
1419
1461
|
account_key = payload.get("account_key")
|
|
@@ -1449,6 +1491,21 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1449
1491
|
merged_payload[field] = proposed_payload[field]
|
|
1450
1492
|
merged = dict(record)
|
|
1451
1493
|
merged["payload"] = merged_payload
|
|
1494
|
+
marker = (historical_close_markers or {}).get(event_id)
|
|
1495
|
+
if marker is not None and not _same_child_population(
|
|
1496
|
+
payload, proposed_payload,
|
|
1497
|
+
):
|
|
1498
|
+
# Hold merging must not hide a failed population proof from the
|
|
1499
|
+
# close resolver. Leave the unmarked frozen fact uncertain.
|
|
1500
|
+
historical_close_markers.pop(event_id, None)
|
|
1501
|
+
marker = None
|
|
1502
|
+
if (marker is not None and not payload.get("_pricing")
|
|
1503
|
+
and payload.get("pricing_provenance_json") is None):
|
|
1504
|
+
# #885: the proof already checked replay's retained population.
|
|
1505
|
+
# Compose its marker with the hold's money-preserving weekly-axis
|
|
1506
|
+
# correction now, rather than waiting for structure to match on
|
|
1507
|
+
# the next invocation. The hold still keeps every monetary byte.
|
|
1508
|
+
merged = _with_close_marker(merged, marker)
|
|
1452
1509
|
desired[event_id] = merged
|
|
1453
1510
|
|
|
1454
1511
|
|
|
@@ -1602,6 +1659,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1602
1659
|
are decided by :func:`_resolve_closed_block_milestones` before the diff,
|
|
1603
1660
|
the same way closes are decided before milestones."""
|
|
1604
1661
|
conflicted_event_ids = frozenset(conflicted_event_ids or ())
|
|
1662
|
+
historical_close_markers = dict(historical_close_markers or {})
|
|
1605
1663
|
desired = _desired_by_id(desired_events)
|
|
1606
1664
|
current = {
|
|
1607
1665
|
event_id: selected.record
|
|
@@ -1615,7 +1673,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1615
1673
|
snapshot_identity_decisions,
|
|
1616
1674
|
)
|
|
1617
1675
|
_preserve_reviewed_hold_dependencies(
|
|
1618
|
-
current, desired, reviewed_weekly_hold_ids,
|
|
1676
|
+
current, desired, reviewed_weekly_hold_ids, historical_close_markers,
|
|
1619
1677
|
)
|
|
1620
1678
|
retained_event_count = sum(
|
|
1621
1679
|
1 for selected in selection.by_id.values()
|
|
@@ -1634,7 +1692,6 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1634
1692
|
for kind in sorted(_EVT_CLASSIFICATIONS)
|
|
1635
1693
|
}
|
|
1636
1694
|
actions: list[PlanAction] = []
|
|
1637
|
-
historical_close_markers = dict(historical_close_markers or {})
|
|
1638
1695
|
drifted_close_ids: set = set()
|
|
1639
1696
|
|
|
1640
1697
|
def _kind(record):
|
|
@@ -1855,8 +1912,8 @@ def _replaced_block_closure(
|
|
|
1855
1912
|
current content, whatever its total), or a money-preserving correction.
|
|
1856
1913
|
A tombstone carries no payload; its block is read from its current
|
|
1857
1914
|
record. Only reviewed actions are ever added, each once. The reverse
|
|
1858
|
-
direction
|
|
1859
|
-
|
|
1915
|
+
direction also applies (#883): an included replacing close brings its
|
|
1916
|
+
milestone actions, even when B and R make those actions identically.
|
|
1860
1917
|
"""
|
|
1861
1918
|
def block(action):
|
|
1862
1919
|
if action.disposition == "tombstone":
|
|
@@ -1902,7 +1959,7 @@ def _replaced_block_closure(
|
|
|
1902
1959
|
replacing.pop(key, None)
|
|
1903
1960
|
triggered = {
|
|
1904
1961
|
block(action) for action in included
|
|
1905
|
-
if action.kind
|
|
1962
|
+
if action.kind in {"five_hour_milestone", "five_hour_block_close"}
|
|
1906
1963
|
and block(action) in replacing
|
|
1907
1964
|
}
|
|
1908
1965
|
if not triggered:
|
|
@@ -1923,6 +1980,152 @@ def _replaced_block_closure(
|
|
|
1923
1980
|
return expanded
|
|
1924
1981
|
|
|
1925
1982
|
|
|
1983
|
+
def _monetary_chain_closure(
|
|
1984
|
+
included: list, reviewed_actions, current_events: Mapping[str, Mapping],
|
|
1985
|
+
) -> list:
|
|
1986
|
+
"""#883: retain the reviewed final predecessor basis of causal marginals.
|
|
1987
|
+
|
|
1988
|
+
An action can change a row's occupied threshold, segment or block cost.
|
|
1989
|
+
The nearest-lower threshold group is then a dependency of every present
|
|
1990
|
+
marginal above it, in both the current and reviewed final populations.
|
|
1991
|
+
Filtering can otherwise leave behind a retired or moved successor with
|
|
1992
|
+
its old marginal. Changed rows entering or leaving either interval are
|
|
1993
|
+
dependencies too: keeping one would select the wrong predecessor.
|
|
1994
|
+
Include dependencies in both directions, even when a
|
|
1995
|
+
successor needs no action itself. A marginal-only correction therefore
|
|
1996
|
+
brings a changed predecessor, but cannot propagate through its unchanged
|
|
1997
|
+
block cost to unrelated siblings. Null marginals have no monetary edge.
|
|
1998
|
+
|
|
1999
|
+
Only reviewed actions are added. Frozen crossing costs have already been
|
|
2000
|
+
settled by the full planner, so closing these edges never reprices them.
|
|
2001
|
+
"""
|
|
2002
|
+
final = dict(current_events)
|
|
2003
|
+
for action in reviewed_actions:
|
|
2004
|
+
if action.disposition == "tombstone":
|
|
2005
|
+
final.pop(action.event_id, None)
|
|
2006
|
+
else:
|
|
2007
|
+
final[action.event_id] = {"payload": action.payload or {}}
|
|
2008
|
+
|
|
2009
|
+
def monetary_position(record):
|
|
2010
|
+
payload = (record or {}).get("payload") or {}
|
|
2011
|
+
if payload.get("kind") != "five_hour_milestone":
|
|
2012
|
+
return None
|
|
2013
|
+
threshold = _usable_threshold(payload.get("percent_threshold"))
|
|
2014
|
+
if threshold is None:
|
|
2015
|
+
return None
|
|
2016
|
+
return (_milestone_segment_key(payload), threshold,
|
|
2017
|
+
payload.get("block_cost_usd"))
|
|
2018
|
+
|
|
2019
|
+
changed = {
|
|
2020
|
+
action.event_id for action in reviewed_actions
|
|
2021
|
+
if action.kind == "five_hour_milestone"
|
|
2022
|
+
and monetary_position(current_events.get(action.event_id))
|
|
2023
|
+
!= monetary_position(final.get(action.event_id))
|
|
2024
|
+
}
|
|
2025
|
+
if not changed:
|
|
2026
|
+
return included
|
|
2027
|
+
def occupied_thresholds(events, event_ids):
|
|
2028
|
+
occupied: dict[tuple, dict[int, set[str]]] = {}
|
|
2029
|
+
for event_id in event_ids:
|
|
2030
|
+
position = monetary_position(events.get(event_id))
|
|
2031
|
+
if position is not None:
|
|
2032
|
+
segment, threshold, _cost = position
|
|
2033
|
+
occupied.setdefault(segment, {}).setdefault(
|
|
2034
|
+
threshold, set()).add(event_id)
|
|
2035
|
+
return occupied
|
|
2036
|
+
|
|
2037
|
+
edges: dict[str, set[str]] = {}
|
|
2038
|
+
for population, other in ((final, current_events), (current_events, final)):
|
|
2039
|
+
occupied = occupied_thresholds(population, population)
|
|
2040
|
+
changed_other = occupied_thresholds(other, changed)
|
|
2041
|
+
for event_id, record in population.items():
|
|
2042
|
+
payload = record.get("payload") or {}
|
|
2043
|
+
position = monetary_position(record)
|
|
2044
|
+
if (position is None or payload.get("marginal_cost_usd") is None
|
|
2045
|
+
or not _usable_amount(payload.get("block_cost_usd"))):
|
|
2046
|
+
continue
|
|
2047
|
+
segment, threshold, _cost = position
|
|
2048
|
+
lower = [value for value in occupied[segment] if value < threshold]
|
|
2049
|
+
nearest = max(lower) if lower else None
|
|
2050
|
+
dependencies = (
|
|
2051
|
+
occupied[segment][nearest] & changed if nearest is not None
|
|
2052
|
+
else set()
|
|
2053
|
+
)
|
|
2054
|
+
for prior_threshold, prior_ids in changed_other.get(segment, {}).items():
|
|
2055
|
+
if (prior_threshold < threshold
|
|
2056
|
+
and (nearest is None or prior_threshold >= nearest)):
|
|
2057
|
+
dependencies.update(prior_ids)
|
|
2058
|
+
if dependencies:
|
|
2059
|
+
edges.setdefault(event_id, set()).update(dependencies)
|
|
2060
|
+
for dependency_id in dependencies:
|
|
2061
|
+
edges.setdefault(dependency_id, set()).add(event_id)
|
|
2062
|
+
# Retained rows participate as dependency nodes too, although there is no
|
|
2063
|
+
# reviewed action to emit for them. Visit each monetary edge once rather
|
|
2064
|
+
# than rescanning the full graph for every later threshold in a chain.
|
|
2065
|
+
required_ids = {action.event_id for action in included}
|
|
2066
|
+
pending = list(required_ids)
|
|
2067
|
+
while pending:
|
|
2068
|
+
event_id = pending.pop()
|
|
2069
|
+
added = edges.get(event_id, set()) - required_ids
|
|
2070
|
+
required_ids.update(added)
|
|
2071
|
+
pending.extend(added)
|
|
2072
|
+
return [action for action in reviewed_actions
|
|
2073
|
+
if action.event_id in required_ids]
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def _causal_snapshot_closure(actions, reviewed_by_id, current_events):
|
|
2077
|
+
"""Close explicit references, keeping a selected dependency if needed."""
|
|
2078
|
+
action_by_id = {action.event_id: action for action in actions}
|
|
2079
|
+
dependency_tombstones = set()
|
|
2080
|
+
for action in actions:
|
|
2081
|
+
if (action.disposition == "tombstone"
|
|
2082
|
+
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2083
|
+
continue
|
|
2084
|
+
for field, expected_kind in _milestone_dependencies(action.kind).items():
|
|
2085
|
+
reference = (action.payload or {}).get(field)
|
|
2086
|
+
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2087
|
+
continue
|
|
2088
|
+
dependency_action = action_by_id.get(reference)
|
|
2089
|
+
if (dependency_action is not None
|
|
2090
|
+
and dependency_action.disposition == "tombstone"):
|
|
2091
|
+
current = current_events.get(reference)
|
|
2092
|
+
if ((current or {}).get("payload") or {}).get("kind") \
|
|
2093
|
+
!= expected_kind:
|
|
2094
|
+
raise RederiveConflict(
|
|
2095
|
+
"causal milestone dependency cannot be retained: "
|
|
2096
|
+
+ action.event_id)
|
|
2097
|
+
dependency_tombstones.add(reference)
|
|
2098
|
+
actions = [action for action in actions
|
|
2099
|
+
if action.event_id not in dependency_tombstones]
|
|
2100
|
+
target_events = dict(current_events)
|
|
2101
|
+
for action in actions:
|
|
2102
|
+
if action.disposition == "tombstone":
|
|
2103
|
+
target_events.pop(action.event_id, None)
|
|
2104
|
+
else:
|
|
2105
|
+
target_events[action.event_id] = {"payload": action.payload or {}}
|
|
2106
|
+
# Appending to this list intentionally visits newly required dependencies.
|
|
2107
|
+
for action in actions:
|
|
2108
|
+
if (action.disposition == "tombstone"
|
|
2109
|
+
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2110
|
+
continue
|
|
2111
|
+
for field, expected_kind in _milestone_dependencies(action.kind).items():
|
|
2112
|
+
reference = (action.payload or {}).get(field)
|
|
2113
|
+
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2114
|
+
continue
|
|
2115
|
+
target = target_events.get(reference)
|
|
2116
|
+
if ((target or {}).get("payload") or {}).get("kind") == expected_kind:
|
|
2117
|
+
continue
|
|
2118
|
+
dependency = reviewed_by_id.get(reference)
|
|
2119
|
+
if (dependency is None or dependency.disposition == "tombstone"
|
|
2120
|
+
or dependency.kind != expected_kind
|
|
2121
|
+
or (dependency.payload or {}).get("kind") != expected_kind):
|
|
2122
|
+
raise RederiveConflict(
|
|
2123
|
+
"causal milestone dependency is missing: " + action.event_id)
|
|
2124
|
+
actions.append(dependency)
|
|
2125
|
+
target_events[reference] = {"payload": dependency.payload or {}}
|
|
2126
|
+
return actions
|
|
2127
|
+
|
|
2128
|
+
|
|
1926
2129
|
def causal_delta_plan(
|
|
1927
2130
|
baseline: RederivePlan, reviewed: RederivePlan, *,
|
|
1928
2131
|
current_events: Mapping[str, Mapping],
|
|
@@ -1939,8 +2142,8 @@ def causal_delta_plan(
|
|
|
1939
2142
|
#875 (spec §4.5): a five-hour block whose close R replaces, by §4.1's
|
|
1940
2143
|
money rule, is applied whole whenever the subset changes one of its
|
|
1941
2144
|
milestones, and a block R holds frozen never is; see
|
|
1942
|
-
:func:`_replaced_block_closure`.
|
|
1943
|
-
|
|
2145
|
+
:func:`_replaced_block_closure`. Monetary predecessor/successor edges and
|
|
2146
|
+
snapshot references are closed with it until the final graph stabilizes.
|
|
1944
2147
|
"""
|
|
1945
2148
|
if (
|
|
1946
2149
|
baseline.journal_high_water != reviewed.journal_high_water
|
|
@@ -1971,73 +2174,18 @@ def causal_delta_plan(
|
|
|
1971
2174
|
)
|
|
1972
2175
|
)
|
|
1973
2176
|
]
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
reference = payload.get(field)
|
|
1987
|
-
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
1988
|
-
continue
|
|
1989
|
-
dependency_action = action_by_id.get(reference)
|
|
1990
|
-
if (dependency_action is not None
|
|
1991
|
-
and dependency_action.disposition == "tombstone"):
|
|
1992
|
-
current = current_events.get(reference)
|
|
1993
|
-
if ((current or {}).get("payload") or {}).get("kind") \
|
|
1994
|
-
!= expected_kind:
|
|
1995
|
-
raise RederiveConflict(
|
|
1996
|
-
"causal milestone dependency cannot be retained: "
|
|
1997
|
-
+ action.event_id
|
|
1998
|
-
)
|
|
1999
|
-
dependency_tombstones.add(reference)
|
|
2000
|
-
if dependency_tombstones:
|
|
2001
|
-
actions = [
|
|
2002
|
-
action for action in actions
|
|
2003
|
-
if action.event_id not in dependency_tombstones
|
|
2004
|
-
]
|
|
2005
|
-
|
|
2006
|
-
target_events = dict(current_events)
|
|
2007
|
-
for action in actions:
|
|
2008
|
-
if action.disposition == "tombstone":
|
|
2009
|
-
target_events.pop(action.event_id, None)
|
|
2010
|
-
else:
|
|
2011
|
-
target_events[action.event_id] = {"payload": action.payload or {}}
|
|
2012
|
-
for action in actions:
|
|
2013
|
-
if (action.disposition == "tombstone"
|
|
2014
|
-
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2015
|
-
continue
|
|
2016
|
-
payload = action.payload or {}
|
|
2017
|
-
for field, expected_kind in _milestone_dependencies(
|
|
2018
|
-
action.kind
|
|
2019
|
-
).items():
|
|
2020
|
-
reference = payload.get(field)
|
|
2021
|
-
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2022
|
-
continue
|
|
2023
|
-
target = target_events.get(reference)
|
|
2024
|
-
if ((target or {}).get("payload") or {}).get("kind") \
|
|
2025
|
-
!= expected_kind:
|
|
2026
|
-
dependency = reviewed_by_id.get(reference)
|
|
2027
|
-
if (
|
|
2028
|
-
dependency is None
|
|
2029
|
-
or dependency.disposition == "tombstone"
|
|
2030
|
-
or dependency.kind != expected_kind
|
|
2031
|
-
or (dependency.payload or {}).get("kind") != expected_kind
|
|
2032
|
-
):
|
|
2033
|
-
raise RederiveConflict(
|
|
2034
|
-
"causal milestone dependency is missing: "
|
|
2035
|
-
+ action.event_id
|
|
2036
|
-
)
|
|
2037
|
-
actions.append(dependency)
|
|
2038
|
-
target_events[reference] = {
|
|
2039
|
-
"payload": dependency.payload or {},
|
|
2040
|
-
}
|
|
2177
|
+
# Close block, monetary and explicit-reference edges to a fixed point.
|
|
2178
|
+
# A dependency introduced by one phase must be seen by every other phase.
|
|
2179
|
+
while True:
|
|
2180
|
+
previous_ids = {action.event_id for action in actions}
|
|
2181
|
+
actions = _replaced_block_closure(
|
|
2182
|
+
actions, reviewed.actions, current_events)
|
|
2183
|
+
actions = _monetary_chain_closure(
|
|
2184
|
+
actions, reviewed.actions, current_events)
|
|
2185
|
+
actions = _causal_snapshot_closure(
|
|
2186
|
+
actions, reviewed_by_id, current_events)
|
|
2187
|
+
if {action.event_id for action in actions} == previous_ids:
|
|
2188
|
+
break
|
|
2041
2189
|
included_ids = {action.event_id for action in actions}
|
|
2042
2190
|
actions = tuple(
|
|
2043
2191
|
action for action in reviewed.actions
|
|
@@ -1315,6 +1315,15 @@ class CodexAccountingCacheResult:
|
|
|
1315
1315
|
#: the generation this capture builds and admits a retained generation
|
|
1316
1316
|
#: only while its stamp equals `codex_accounting_consumed_provenance()`.
|
|
1317
1317
|
provenance_token: int | None = None
|
|
1318
|
+
#: #872: the NAME of the population `changed_old` / `changed_new` are
|
|
1319
|
+
#: relative to — the prior state's `population_signature` and provenance
|
|
1320
|
+
#: token, read before this call mutated the state. `None` on every COLD
|
|
1321
|
+
#: result: a cold delta replaces the whole population relative to nothing
|
|
1322
|
+
#: a consumer can name (an empty state, a reset, an overflow clear, or a
|
|
1323
|
+
#: semantic/start/end/sequence/overflow cold trigger), so a consumer must
|
|
1324
|
+
#: derive from the full population instead of advancing retained state.
|
|
1325
|
+
base_population_signature: tuple | None = None
|
|
1326
|
+
base_provenance_token: int | None = None
|
|
1318
1327
|
|
|
1319
1328
|
|
|
1320
1329
|
_CODEX_ACCOUNTING_CACHE_STATE: dict[str, object] = _ObservedSnapshotDict()
|
|
@@ -1551,6 +1560,14 @@ def build_cached_codex_accounting(
|
|
|
1551
1560
|
attribution_revision = _codex_window_attribution_revision(cache_conn)
|
|
1552
1561
|
state = _CODEX_ACCOUNTING_CACHE_STATE
|
|
1553
1562
|
prior_cached_entries = tuple(state.get("entries", ())) if state else ()
|
|
1563
|
+
# #872: the delta's base, read before any branch below mutates the state.
|
|
1564
|
+
prior_signature = state.get("population_signature") if state else None
|
|
1565
|
+
if not isinstance(prior_signature, tuple):
|
|
1566
|
+
prior_signature = None
|
|
1567
|
+
prior_token = (
|
|
1568
|
+
state.get(_CODEX_ACCOUNTING_PROVENANCE_STATE_KEY) if state else None)
|
|
1569
|
+
if not isinstance(prior_token, int):
|
|
1570
|
+
prior_token = None
|
|
1554
1571
|
|
|
1555
1572
|
def signature(generation: int) -> tuple | None:
|
|
1556
1573
|
"""The exact identity of the population this call is returning.
|
|
@@ -1683,6 +1700,8 @@ def build_cached_codex_accounting(
|
|
|
1683
1700
|
population_signature=population_signature,
|
|
1684
1701
|
provenance_token=_consume_codex_accounting_provenance(
|
|
1685
1702
|
provenance_token),
|
|
1703
|
+
base_population_signature=prior_signature,
|
|
1704
|
+
base_provenance_token=prior_token,
|
|
1686
1705
|
)
|
|
1687
1706
|
|
|
1688
1707
|
dirty_paths = tuple(sorted(dirty))
|
|
@@ -1736,6 +1755,8 @@ def build_cached_codex_accounting(
|
|
|
1736
1755
|
entries, dirty_paths, dirty_accounts, False, changed_old, changed_new,
|
|
1737
1756
|
population_signature=population_signature,
|
|
1738
1757
|
provenance_token=_consume_codex_accounting_provenance(provenance_token),
|
|
1758
|
+
base_population_signature=prior_signature,
|
|
1759
|
+
base_provenance_token=prior_token,
|
|
1739
1760
|
)
|
|
1740
1761
|
|
|
1741
1762
|
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Per-connection SQLite checkpoint-on-close control (#901 spec §5.4).
|
|
2
|
+
|
|
3
|
+
The compatibility helper that `_cctally_cache._set_cache_no_checkpoint_on_close`
|
|
4
|
+
introduced for cache recovery, generalized to any connection. Python 3.12+
|
|
5
|
+
reaches `SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE` through `Connection.setconfig`;
|
|
6
|
+
CPython 3.11 reaches the same SQLite API through its supported
|
|
7
|
+
`pysqlite_Connection` layout and the stdlib extension's own linked
|
|
8
|
+
`sqlite3_db_config`, so a different SQLite instance is never bound.
|
|
9
|
+
Stdlib-only leaf.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import sqlite3
|
|
14
|
+
import sys
|
|
15
|
+
|
|
16
|
+
#: `SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE` from sqlite3.h.
|
|
17
|
+
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE = 1006
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_no_checkpoint_on_close(conn: sqlite3.Connection, disabled: bool, *,
|
|
21
|
+
purpose: str = "connection") -> None:
|
|
22
|
+
"""Turn SQLite's checkpoint at the last close off (``disabled=True``) or
|
|
23
|
+
back on; raise `sqlite3.NotSupportedError` when it cannot be applied."""
|
|
24
|
+
option = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
|
|
25
|
+
setconfig = getattr(conn, "setconfig", None)
|
|
26
|
+
if option is not None and setconfig is not None:
|
|
27
|
+
setconfig(option, bool(disabled))
|
|
28
|
+
getconfig = getattr(conn, "getconfig", None)
|
|
29
|
+
if getconfig is not None and bool(getconfig(option)) != bool(disabled):
|
|
30
|
+
raise sqlite3.NotSupportedError(
|
|
31
|
+
f"{purpose} could not configure SQLite close checkpointing")
|
|
32
|
+
return
|
|
33
|
+
set_no_checkpoint_on_close_cpython(conn, disabled, purpose=purpose)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def set_no_checkpoint_on_close_cpython(conn: sqlite3.Connection,
|
|
37
|
+
disabled: bool, *,
|
|
38
|
+
purpose: str = "connection") -> None:
|
|
39
|
+
"""Python 3.11 compatibility adapter for sqlite3_db_config()."""
|
|
40
|
+
if sys.implementation.name != "cpython":
|
|
41
|
+
raise sqlite3.NotSupportedError(
|
|
42
|
+
f"{purpose} requires SQLite no-checkpoint-on-close support")
|
|
43
|
+
|
|
44
|
+
# CPython 3.11's public sqlite3 module does not expose db_config(), but its
|
|
45
|
+
# connection layout begins with PyObject_HEAD followed by ``sqlite3 *db``.
|
|
46
|
+
# Load sqlite3_db_config from the same extension so we never bind a
|
|
47
|
+
# different SQLite instance.
|
|
48
|
+
import _sqlite3
|
|
49
|
+
import ctypes
|
|
50
|
+
|
|
51
|
+
sqlite_lib = ctypes.CDLL(_sqlite3.__file__)
|
|
52
|
+
db_config = sqlite_lib.sqlite3_db_config
|
|
53
|
+
db_config.argtypes = (ctypes.c_void_p, ctypes.c_int)
|
|
54
|
+
db_config.restype = ctypes.c_int
|
|
55
|
+
pointer_size = ctypes.sizeof(ctypes.c_void_p)
|
|
56
|
+
db_pointer = ctypes.c_void_p.from_address(
|
|
57
|
+
id(conn) + (2 * pointer_size)
|
|
58
|
+
).value
|
|
59
|
+
if not db_pointer:
|
|
60
|
+
raise sqlite3.NotSupportedError(
|
|
61
|
+
f"{purpose} could not resolve the SQLite connection handle")
|
|
62
|
+
current = ctypes.c_int()
|
|
63
|
+
rc = db_config(
|
|
64
|
+
ctypes.c_void_p(db_pointer),
|
|
65
|
+
SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,
|
|
66
|
+
ctypes.c_int(1 if disabled else 0),
|
|
67
|
+
ctypes.byref(current),
|
|
68
|
+
)
|
|
69
|
+
if rc != sqlite3.SQLITE_OK or current.value != int(bool(disabled)):
|
|
70
|
+
raise sqlite3.NotSupportedError(
|
|
71
|
+
f"{purpose} could not configure SQLite close checkpointing")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def no_checkpoint_on_close_enabled(conn: sqlite3.Connection) -> "bool | None":
|
|
75
|
+
"""The setting as SQLite reports it, or None where it cannot be read
|
|
76
|
+
(CPython 3.11, whose adapter verifies at set time instead)."""
|
|
77
|
+
option = getattr(sqlite3, "SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE", None)
|
|
78
|
+
getconfig = getattr(conn, "getconfig", None)
|
|
79
|
+
if option is None or getconfig is None:
|
|
80
|
+
return None
|
|
81
|
+
return bool(getconfig(option))
|
|
@@ -504,6 +504,105 @@ def subscription_window_ending_at(
|
|
|
504
504
|
return window
|
|
505
505
|
|
|
506
506
|
|
|
507
|
+
#: The pre-#901 grouped anchor read, kept as the fallback for a stats index
|
|
508
|
+
#: without the epoch-1017 anchor indexes (it selects the same rows, but sorts
|
|
509
|
+
#: one group per subscription week ever recorded through a temp b-tree).
|
|
510
|
+
_SUBSCRIPTION_ANCHOR_GROUPED_SQL = (
|
|
511
|
+
"SELECT "
|
|
512
|
+
" MIN(week_start_at) AS week_start_at, "
|
|
513
|
+
" MIN(week_end_at) AS week_end_at, "
|
|
514
|
+
" week_start_date, "
|
|
515
|
+
" MIN(week_end_date) AS week_end_date "
|
|
516
|
+
"FROM weekly_usage_snapshots "
|
|
517
|
+
"WHERE week_start_at IS NOT NULL "
|
|
518
|
+
" AND week_end_at IS NOT NULL "
|
|
519
|
+
" AND week_start_date IS NOT NULL "
|
|
520
|
+
" {account} "
|
|
521
|
+
"GROUP BY week_start_date "
|
|
522
|
+
"ORDER BY MIN(week_start_at) ASC"
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def _subscription_anchor_eligibility(prefix: str) -> str:
|
|
527
|
+
return (
|
|
528
|
+
f"{prefix}week_start_at IS NOT NULL "
|
|
529
|
+
f"AND {prefix}week_end_at IS NOT NULL "
|
|
530
|
+
f"AND {prefix}week_start_date IS NOT NULL"
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _subscription_anchor_sql(scoped: bool) -> str:
|
|
535
|
+
"""The ordered representative-row anchor read (#901 Amendment 1 item 9).
|
|
536
|
+
|
|
537
|
+
The outer scan walks every qualifying row in ``(week_start_at,
|
|
538
|
+
week_start_date)`` order through ``idx_usage_subscription_anchor_order``
|
|
539
|
+
and keeps one row per date: the date's first qualifying row by
|
|
540
|
+
``(week_start_at, id)``, found with one seek of
|
|
541
|
+
``idx_usage_subscription_anchor_pick``. That row's ``week_start_at`` is the
|
|
542
|
+
date's lexical ``MIN(week_start_at)``, so rows leave in the old ``ORDER BY
|
|
543
|
+
MIN(week_start_at)`` order, with equal starts on two dates in date order.
|
|
544
|
+
``MIN(week_end_at)`` and ``MIN(week_end_date)`` are INDEPENDENT correlated
|
|
545
|
+
aggregates over the date's complete qualifying population, because the
|
|
546
|
+
three minima can come from three different rows. The eligibility and the
|
|
547
|
+
optional account predicate repeat in the outer scan and in every lookup.
|
|
548
|
+
"""
|
|
549
|
+
def account(alias: str) -> str:
|
|
550
|
+
return f" AND {alias}.account_key = ?" if scoped else ""
|
|
551
|
+
|
|
552
|
+
def lookup(alias: str) -> str:
|
|
553
|
+
return (
|
|
554
|
+
f"FROM weekly_usage_snapshots AS {alias} "
|
|
555
|
+
"INDEXED BY idx_usage_subscription_anchor_pick "
|
|
556
|
+
f"WHERE {alias}.week_start_date = o.week_start_date "
|
|
557
|
+
f"AND {_subscription_anchor_eligibility(alias + '.')}"
|
|
558
|
+
f"{account(alias)}"
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
return (
|
|
562
|
+
"SELECT o.week_start_at AS week_start_at, "
|
|
563
|
+
f"(SELECT MIN(e.week_end_at) {lookup('e')}) AS week_end_at, "
|
|
564
|
+
"o.week_start_date AS week_start_date, "
|
|
565
|
+
f"(SELECT MIN(d.week_end_date) {lookup('d')}) AS week_end_date "
|
|
566
|
+
"FROM weekly_usage_snapshots AS o "
|
|
567
|
+
"INDEXED BY idx_usage_subscription_anchor_order "
|
|
568
|
+
f"WHERE {_subscription_anchor_eligibility('o.')}{account('o')} "
|
|
569
|
+
f"AND o.id = (SELECT p.id {lookup('p')} "
|
|
570
|
+
"ORDER BY p.week_start_at ASC, p.id ASC LIMIT 1) "
|
|
571
|
+
"ORDER BY o.week_start_at ASC, o.week_start_date ASC"
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _subscription_anchor_rows(
|
|
576
|
+
conn: sqlite3.Connection, account_key: "str | None",
|
|
577
|
+
) -> list:
|
|
578
|
+
"""One ``(start_at, end_at, start_date, end_date)`` row per subscription
|
|
579
|
+
week, ascending by the week's minimum start (#901 Amendment 1 item 9).
|
|
580
|
+
|
|
581
|
+
The pre-#901 statement grouped every qualifying snapshot by date and
|
|
582
|
+
sorted the groups by ``MIN(week_start_at)`` through temp b-trees, with no
|
|
583
|
+
bound on retained weeks; ``_subscription_anchor_sql`` returns the same
|
|
584
|
+
rows by streaming two epoch-1017 indexes (``c901-design-2``). A store
|
|
585
|
+
without both indexes keeps the grouped statement, which is correct.
|
|
586
|
+
"""
|
|
587
|
+
present = {
|
|
588
|
+
str(row[0]) for row in conn.execute(
|
|
589
|
+
"SELECT name FROM sqlite_master WHERE type='index' AND name IN "
|
|
590
|
+
"('idx_usage_subscription_anchor_order', "
|
|
591
|
+
"'idx_usage_subscription_anchor_pick')")
|
|
592
|
+
}
|
|
593
|
+
params: tuple = () if account_key is None else (account_key,)
|
|
594
|
+
if len(present) == 2:
|
|
595
|
+
# The account placeholder appears four times: the two aggregates, the
|
|
596
|
+
# outer scan and the representative pick, in that textual order.
|
|
597
|
+
return conn.execute(
|
|
598
|
+
_subscription_anchor_sql(account_key is not None), params * 4,
|
|
599
|
+
).fetchall()
|
|
600
|
+
account = "" if account_key is None else " AND account_key = ?"
|
|
601
|
+
return conn.execute(
|
|
602
|
+
_SUBSCRIPTION_ANCHOR_GROUPED_SQL.format(account=account), params,
|
|
603
|
+
).fetchall()
|
|
604
|
+
|
|
605
|
+
|
|
507
606
|
def _compute_subscription_weeks(
|
|
508
607
|
conn: sqlite3.Connection,
|
|
509
608
|
range_start: dt.datetime,
|
|
@@ -545,23 +644,7 @@ def _compute_subscription_weeks(
|
|
|
545
644
|
"""
|
|
546
645
|
# Case A: snapshots exist. Account scoping (#341): a real key filters the
|
|
547
646
|
# reset anchors to that account; None is the merged (all-accounts) read.
|
|
548
|
-
|
|
549
|
-
acct_params: tuple = () if account_key is None else (account_key,)
|
|
550
|
-
snap_rows = conn.execute(
|
|
551
|
-
"SELECT "
|
|
552
|
-
" MIN(week_start_at) AS week_start_at, "
|
|
553
|
-
" MIN(week_end_at) AS week_end_at, "
|
|
554
|
-
" week_start_date, "
|
|
555
|
-
" MIN(week_end_date) AS week_end_date "
|
|
556
|
-
"FROM weekly_usage_snapshots "
|
|
557
|
-
"WHERE week_start_at IS NOT NULL "
|
|
558
|
-
" AND week_end_at IS NOT NULL "
|
|
559
|
-
" AND week_start_date IS NOT NULL "
|
|
560
|
-
f" {acct_pred} "
|
|
561
|
-
"GROUP BY week_start_date "
|
|
562
|
-
"ORDER BY MIN(week_start_at) ASC",
|
|
563
|
-
acct_params,
|
|
564
|
-
).fetchall()
|
|
647
|
+
snap_rows = _subscription_anchor_rows(conn, account_key)
|
|
565
648
|
|
|
566
649
|
weeks: list[SubWeek] = []
|
|
567
650
|
|