cctally 1.113.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +6035 -722
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +400 -2
- package/bin/_cctally_dashboard.py +98 -20
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +285 -213
- package/bin/_cctally_db.py +1177 -23
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +572 -45
- package/bin/_cctally_five_hour.py +84 -33
- package/bin/_cctally_forecast.py +108 -15
- package/bin/_cctally_journal.py +3950 -116
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +227 -37
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +691 -149
- package/bin/_cctally_rederive.py +320 -41
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +83 -17
- package/bin/_cctally_statusline.py +6154 -517
- package/bin/_cctally_store.py +367 -53
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +61 -56
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +297 -49
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +106 -59
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +714 -69
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +231 -86
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +199 -9
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +319 -82
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +31 -1
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +19 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_lib_rederive.py
CHANGED
|
@@ -144,6 +144,24 @@ _OP_CLASSIFICATIONS = {
|
|
|
144
144
|
"retained", "operator Codex window attribution is outside claude-usage"),
|
|
145
145
|
"codex_window_attribution_retract": KindClassification(
|
|
146
146
|
"retained", "operator Codex window retraction is outside claude-usage"),
|
|
147
|
+
# #825/#832 spec 5.1. The versioned reviewed decision is replayed into
|
|
148
|
+
# scratch exactly like its previous-release twin.
|
|
149
|
+
"claude_weekly_observation_decision_v2": KindClassification(
|
|
150
|
+
"retained_input",
|
|
151
|
+
"reviewed weekly-axis decisions are replayed into scratch"),
|
|
152
|
+
# The account-bound protocol ops are fence metadata: they decide what is
|
|
153
|
+
# admitted and anchor the legacy spool baseline, and nothing in the
|
|
154
|
+
# claude-usage family derives them.
|
|
155
|
+
"account_bound_cutover": KindClassification(
|
|
156
|
+
"retained", "the account-bound journal fence is replayed, never "
|
|
157
|
+
"re-derived"),
|
|
158
|
+
"legacy_spool_activity": KindClassification(
|
|
159
|
+
"retained", "legacy spool evidence is replayed, never re-derived"),
|
|
160
|
+
# The versioned journal-repair acknowledgement is an audit record over the
|
|
161
|
+
# raw prefix, outside claude-usage (its previous-release twin is consumed
|
|
162
|
+
# by the selector directly and never registered).
|
|
163
|
+
"journal_protocol_resolution_v2": KindClassification(
|
|
164
|
+
"retained", "journal-repair acknowledgements are audit records"),
|
|
147
165
|
}
|
|
148
166
|
|
|
149
167
|
|
|
@@ -194,7 +212,15 @@ def validate_family_registry(*, evt_kinds: set[str],
|
|
|
194
212
|
)
|
|
195
213
|
|
|
196
214
|
|
|
215
|
+
# #874: the contract is structural. It names every column any planner reader
|
|
216
|
+
# of the table reads (`_validate_cache_rows`, `_cache_fingerprint`,
|
|
217
|
+
# `_joined_entries`, `iter_entries`), not only those the current journal
|
|
218
|
+
# reaches, so whether a cache is accepted never depends on journal or pricing
|
|
219
|
+
# state, and a missing column is a typed gap rather than a raw sqlite error.
|
|
197
220
|
_SESSION_ENTRY_COLUMNS = frozenset({
|
|
221
|
+
"id",
|
|
222
|
+
"source_path",
|
|
223
|
+
"line_offset",
|
|
198
224
|
"timestamp_utc",
|
|
199
225
|
"model",
|
|
200
226
|
"input_tokens",
|
|
@@ -202,7 +228,8 @@ _SESSION_ENTRY_COLUMNS = frozenset({
|
|
|
202
228
|
"cache_create_tokens",
|
|
203
229
|
"cache_read_tokens",
|
|
204
230
|
"cache_create_1h_tokens",
|
|
205
|
-
"
|
|
231
|
+
"cost_usd_raw",
|
|
232
|
+
"speed",
|
|
206
233
|
"account_key",
|
|
207
234
|
})
|
|
208
235
|
_SESSION_FILE_COLUMNS = frozenset({"path", "session_id", "project_path"})
|
|
@@ -228,6 +255,37 @@ def validate_claude_cache_contract(tables: Mapping[str, set[str]]) -> None:
|
|
|
228
255
|
)
|
|
229
256
|
|
|
230
257
|
|
|
258
|
+
# #874: the Codex columns `_cache_fingerprint(include_codex=True)` and the
|
|
259
|
+
# Codex budget re-pricing reader (`iter_codex_entries`) read. Required only
|
|
260
|
+
# when the planner reads the Codex cache at all.
|
|
261
|
+
_CODEX_SESSION_ENTRY_COLUMNS = frozenset({
|
|
262
|
+
"source_path",
|
|
263
|
+
"line_offset",
|
|
264
|
+
"timestamp_utc",
|
|
265
|
+
"session_id",
|
|
266
|
+
"model",
|
|
267
|
+
"input_tokens",
|
|
268
|
+
"cached_input_tokens",
|
|
269
|
+
"output_tokens",
|
|
270
|
+
"reasoning_output_tokens",
|
|
271
|
+
"total_tokens",
|
|
272
|
+
"account_key",
|
|
273
|
+
})
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def validate_codex_cache_contract(tables: Mapping[str, set[str]]) -> None:
|
|
277
|
+
"""Fail before planning if the Codex cost inputs the planner reads are gone."""
|
|
278
|
+
if "codex_session_entries" not in tables:
|
|
279
|
+
raise RederiveDataGap("missing cache.db table codex_session_entries")
|
|
280
|
+
missing = sorted(
|
|
281
|
+
_CODEX_SESSION_ENTRY_COLUMNS - set(tables["codex_session_entries"]))
|
|
282
|
+
if missing:
|
|
283
|
+
raise RederiveDataGap(
|
|
284
|
+
"missing cache.db codex_session_entries column(s): "
|
|
285
|
+
+ ", ".join(missing)
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
|
|
231
289
|
def preserved_history(records: Iterable[Mapping], *,
|
|
232
290
|
evidence_retained: bool) -> dict[str, Mapping]:
|
|
233
291
|
"""Owned events the current re-derivation cannot reach, keyed by event id.
|
|
@@ -1270,9 +1328,48 @@ def _reconcile_snapshot_identities(
|
|
|
1270
1328
|
)
|
|
1271
1329
|
|
|
1272
1330
|
|
|
1331
|
+
def _suppression_deletes_snapshot(record: Mapping, *, weekly: bool,
|
|
1332
|
+
five_hour: bool) -> bool:
|
|
1333
|
+
"""Whether the stale-replica suppressions naming one snapshot delete it
|
|
1334
|
+
(#825/#832 spec 4.7 revisions 6 and 7), as the journal appliers
|
|
1335
|
+
materialize them. A five-hour credit retires an unheld row's five-hour
|
|
1336
|
+
reading in place and deletes a held row; a weekly suppression makes a row
|
|
1337
|
+
that carries its own five-hour reading weekly-held and deletes a row that
|
|
1338
|
+
keeps no evidence on either axis. Both retirements together delete the
|
|
1339
|
+
row, in either order."""
|
|
1340
|
+
payload = record.get("payload") or {}
|
|
1341
|
+
held = payload.get("weekly_observation_held") == 1
|
|
1342
|
+
carries_five_hour = (
|
|
1343
|
+
payload.get("five_hour_percent") is not None
|
|
1344
|
+
and payload.get("five_hour_resets_at") is not None
|
|
1345
|
+
and payload.get("source") != "record-credit"
|
|
1346
|
+
)
|
|
1347
|
+
if weekly and five_hour:
|
|
1348
|
+
return True
|
|
1349
|
+
if five_hour:
|
|
1350
|
+
return held
|
|
1351
|
+
if weekly:
|
|
1352
|
+
return not carries_five_hour
|
|
1353
|
+
return False
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def _without_snapshot_suppression(payload: Mapping) -> Mapping:
|
|
1357
|
+
"""``payload`` without its `weekly_usage_snapshots` stale-replica
|
|
1358
|
+
suppression list. Since spec 4.7 revision 7 (fix round K16b) a
|
|
1359
|
+
suppression that names a reviewed-held snapshot can leave it
|
|
1360
|
+
materialized (a weekly credit keeps a row carrying its own five-hour
|
|
1361
|
+
reading weekly-held), and that list is then a retirement effect on the
|
|
1362
|
+
snapshot, not a reference that depends on it."""
|
|
1363
|
+
if ("suppression" not in payload
|
|
1364
|
+
or payload.get("suppression_table", "weekly_usage_snapshots")
|
|
1365
|
+
!= "weekly_usage_snapshots"):
|
|
1366
|
+
return payload
|
|
1367
|
+
return {key: value for key, value in payload.items() if key != "suppression"}
|
|
1368
|
+
|
|
1369
|
+
|
|
1273
1370
|
def _preserve_reviewed_hold_dependencies(
|
|
1274
1371
|
current: Mapping[str, Mapping], desired: dict[str, Mapping],
|
|
1275
|
-
reviewed_weekly_hold_ids=(),
|
|
1372
|
+
reviewed_weekly_hold_ids=(), historical_close_markers=None,
|
|
1276
1373
|
) -> None:
|
|
1277
1374
|
"""Keep frozen five-hour facts reached from exact reviewed holds.
|
|
1278
1375
|
|
|
@@ -1294,9 +1391,11 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1294
1391
|
}
|
|
1295
1392
|
if not snapshot_ids:
|
|
1296
1393
|
return
|
|
1297
|
-
|
|
1394
|
+
weekly_named: set = set()
|
|
1395
|
+
five_hour_named: set = set()
|
|
1298
1396
|
for record in (*current.values(), *desired.values()):
|
|
1299
|
-
|
|
1397
|
+
payload = record.get("payload") or {}
|
|
1398
|
+
suppression = payload.get("suppression")
|
|
1300
1399
|
if suppression is None:
|
|
1301
1400
|
continue
|
|
1302
1401
|
if not isinstance(suppression, list) or any(
|
|
@@ -1306,7 +1405,26 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1306
1405
|
"reviewed weekly hold suppression has an unfamiliar shape: "
|
|
1307
1406
|
+ str(record.get("id"))
|
|
1308
1407
|
)
|
|
1309
|
-
|
|
1408
|
+
named = snapshot_ids & set(suppression)
|
|
1409
|
+
if (payload.get("suppression_table", "weekly_usage_snapshots")
|
|
1410
|
+
!= "weekly_usage_snapshots"):
|
|
1411
|
+
continue
|
|
1412
|
+
if payload.get("kind") == "five_hour_credit":
|
|
1413
|
+
five_hour_named.update(named)
|
|
1414
|
+
else:
|
|
1415
|
+
weekly_named.update(named)
|
|
1416
|
+
# #825/#832 spec 4.7 revision 7 (*Versioning and replay*, fix round
|
|
1417
|
+
# K16b): suppression-list membership no longer implies whole-snapshot
|
|
1418
|
+
# deletion. Each axis's stale-replica suppression retires its own axis
|
|
1419
|
+
# (`_cctally_journal._apply_reset_with_suppression`), so a snapshot is
|
|
1420
|
+
# deleted only where the materialization deletes it.
|
|
1421
|
+
suppressed_snapshot_ids = {
|
|
1422
|
+
snapshot_id for snapshot_id in weekly_named | five_hour_named
|
|
1423
|
+
if _suppression_deletes_snapshot(
|
|
1424
|
+
(desired.get(snapshot_id) or current.get(snapshot_id) or {}),
|
|
1425
|
+
weekly=snapshot_id in weekly_named,
|
|
1426
|
+
five_hour=snapshot_id in five_hour_named)
|
|
1427
|
+
}
|
|
1310
1428
|
if suppressed_snapshot_ids:
|
|
1311
1429
|
for event_id, record in (*current.items(), *desired.items()):
|
|
1312
1430
|
payload = record.get("payload") or {}
|
|
@@ -1328,7 +1446,8 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1328
1446
|
payload = record.get("payload") or {}
|
|
1329
1447
|
usage_ref = payload.get("usage_snapshot_ref")
|
|
1330
1448
|
if usage_ref not in snapshot_ids:
|
|
1331
|
-
if _contains_any_value(payload,
|
|
1449
|
+
if _contains_any_value(_without_snapshot_suppression(payload),
|
|
1450
|
+
snapshot_ids):
|
|
1332
1451
|
raise RederiveConflict(
|
|
1333
1452
|
"reviewed weekly hold dependency has an unfamiliar shape: "
|
|
1334
1453
|
+ event_id
|
|
@@ -1373,7 +1492,8 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1373
1492
|
payload = record.get("payload") or {}
|
|
1374
1493
|
usage_ref = payload.get("usage_snapshot_ref")
|
|
1375
1494
|
if usage_ref not in snapshot_ids:
|
|
1376
|
-
if _contains_any_value(payload,
|
|
1495
|
+
if _contains_any_value(_without_snapshot_suppression(payload),
|
|
1496
|
+
snapshot_ids):
|
|
1377
1497
|
raise RederiveConflict(
|
|
1378
1498
|
"reviewed weekly hold dependency has an unfamiliar shape: "
|
|
1379
1499
|
+ event_id
|
|
@@ -1413,7 +1533,9 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1413
1533
|
|
|
1414
1534
|
held_windows = set()
|
|
1415
1535
|
for snapshot_id in snapshot_ids:
|
|
1416
|
-
|
|
1536
|
+
# #885: only final support may freeze a close. A current snapshot
|
|
1537
|
+
# absent from this dependency-closed target is retired by this plan.
|
|
1538
|
+
payload = (desired.get(snapshot_id) or {}).get("payload") or {}
|
|
1417
1539
|
if payload.get("kind") != "snapshot_accept":
|
|
1418
1540
|
continue
|
|
1419
1541
|
account_key = payload.get("account_key")
|
|
@@ -1449,6 +1571,21 @@ def _preserve_reviewed_hold_dependencies(
|
|
|
1449
1571
|
merged_payload[field] = proposed_payload[field]
|
|
1450
1572
|
merged = dict(record)
|
|
1451
1573
|
merged["payload"] = merged_payload
|
|
1574
|
+
marker = (historical_close_markers or {}).get(event_id)
|
|
1575
|
+
if marker is not None and not _same_child_population(
|
|
1576
|
+
payload, proposed_payload,
|
|
1577
|
+
):
|
|
1578
|
+
# Hold merging must not hide a failed population proof from the
|
|
1579
|
+
# close resolver. Leave the unmarked frozen fact uncertain.
|
|
1580
|
+
historical_close_markers.pop(event_id, None)
|
|
1581
|
+
marker = None
|
|
1582
|
+
if (marker is not None and not payload.get("_pricing")
|
|
1583
|
+
and payload.get("pricing_provenance_json") is None):
|
|
1584
|
+
# #885: the proof already checked replay's retained population.
|
|
1585
|
+
# Compose its marker with the hold's money-preserving weekly-axis
|
|
1586
|
+
# correction now, rather than waiting for structure to match on
|
|
1587
|
+
# the next invocation. The hold still keeps every monetary byte.
|
|
1588
|
+
merged = _with_close_marker(merged, marker)
|
|
1452
1589
|
desired[event_id] = merged
|
|
1453
1590
|
|
|
1454
1591
|
|
|
@@ -1568,6 +1705,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1568
1705
|
reviewed_weekly_hold_ids=(),
|
|
1569
1706
|
uncertain_cost_fact_ids=(),
|
|
1570
1707
|
historical_close_markers=None,
|
|
1708
|
+
physical_revisions=None,
|
|
1571
1709
|
enforce_guard=True) -> RederivePlan:
|
|
1572
1710
|
"""Diff current effective events against one scratch-derived desired set.
|
|
1573
1711
|
|
|
@@ -1602,6 +1740,12 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1602
1740
|
are decided by :func:`_resolve_closed_block_milestones` before the diff,
|
|
1603
1741
|
the same way closes are decided before milestones."""
|
|
1604
1742
|
conflicted_event_ids = frozenset(conflicted_event_ids or ())
|
|
1743
|
+
historical_close_markers = dict(historical_close_markers or {})
|
|
1744
|
+
# #825/#832 fix round A (Codex fix-round-1 Q2): the highest revision of
|
|
1745
|
+
# every physical record admission hides (an excluded legacy event or
|
|
1746
|
+
# correction action). A new revision is allocated above it, so the
|
|
1747
|
+
# previous release never reads a same-revision conflict.
|
|
1748
|
+
physical_revisions = dict(physical_revisions or {})
|
|
1605
1749
|
desired = _desired_by_id(desired_events)
|
|
1606
1750
|
current = {
|
|
1607
1751
|
event_id: selected.record
|
|
@@ -1615,7 +1759,7 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1615
1759
|
snapshot_identity_decisions,
|
|
1616
1760
|
)
|
|
1617
1761
|
_preserve_reviewed_hold_dependencies(
|
|
1618
|
-
current, desired, reviewed_weekly_hold_ids,
|
|
1762
|
+
current, desired, reviewed_weekly_hold_ids, historical_close_markers,
|
|
1619
1763
|
)
|
|
1620
1764
|
retained_event_count = sum(
|
|
1621
1765
|
1 for selected in selection.by_id.values()
|
|
@@ -1634,7 +1778,6 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1634
1778
|
for kind in sorted(_EVT_CLASSIFICATIONS)
|
|
1635
1779
|
}
|
|
1636
1780
|
actions: list[PlanAction] = []
|
|
1637
|
-
historical_close_markers = dict(historical_close_markers or {})
|
|
1638
1781
|
drifted_close_ids: set = set()
|
|
1639
1782
|
|
|
1640
1783
|
def _kind(record):
|
|
@@ -1767,6 +1910,9 @@ def build_claude_usage_plan(*, selection, desired_events: Iterable[Mapping],
|
|
|
1767
1910
|
disposition = "add"
|
|
1768
1911
|
at = str(desired_record["at"])
|
|
1769
1912
|
payload = dict(desired_record.get("payload") or {})
|
|
1913
|
+
floor = physical_revisions.get(event_id)
|
|
1914
|
+
if floor is not None and revision <= floor:
|
|
1915
|
+
revision = floor + 1
|
|
1770
1916
|
counts[disposition] += 1
|
|
1771
1917
|
kind_record = (
|
|
1772
1918
|
current_record if disposition == "tombstone"
|
|
@@ -1855,8 +2001,8 @@ def _replaced_block_closure(
|
|
|
1855
2001
|
current content, whatever its total), or a money-preserving correction.
|
|
1856
2002
|
A tombstone carries no payload; its block is read from its current
|
|
1857
2003
|
record. Only reviewed actions are ever added, each once. The reverse
|
|
1858
|
-
direction
|
|
1859
|
-
|
|
2004
|
+
direction also applies (#883): an included replacing close brings its
|
|
2005
|
+
milestone actions, even when B and R make those actions identically.
|
|
1860
2006
|
"""
|
|
1861
2007
|
def block(action):
|
|
1862
2008
|
if action.disposition == "tombstone":
|
|
@@ -1902,7 +2048,7 @@ def _replaced_block_closure(
|
|
|
1902
2048
|
replacing.pop(key, None)
|
|
1903
2049
|
triggered = {
|
|
1904
2050
|
block(action) for action in included
|
|
1905
|
-
if action.kind
|
|
2051
|
+
if action.kind in {"five_hour_milestone", "five_hour_block_close"}
|
|
1906
2052
|
and block(action) in replacing
|
|
1907
2053
|
}
|
|
1908
2054
|
if not triggered:
|
|
@@ -1923,6 +2069,152 @@ def _replaced_block_closure(
|
|
|
1923
2069
|
return expanded
|
|
1924
2070
|
|
|
1925
2071
|
|
|
2072
|
+
def _monetary_chain_closure(
|
|
2073
|
+
included: list, reviewed_actions, current_events: Mapping[str, Mapping],
|
|
2074
|
+
) -> list:
|
|
2075
|
+
"""#883: retain the reviewed final predecessor basis of causal marginals.
|
|
2076
|
+
|
|
2077
|
+
An action can change a row's occupied threshold, segment or block cost.
|
|
2078
|
+
The nearest-lower threshold group is then a dependency of every present
|
|
2079
|
+
marginal above it, in both the current and reviewed final populations.
|
|
2080
|
+
Filtering can otherwise leave behind a retired or moved successor with
|
|
2081
|
+
its old marginal. Changed rows entering or leaving either interval are
|
|
2082
|
+
dependencies too: keeping one would select the wrong predecessor.
|
|
2083
|
+
Include dependencies in both directions, even when a
|
|
2084
|
+
successor needs no action itself. A marginal-only correction therefore
|
|
2085
|
+
brings a changed predecessor, but cannot propagate through its unchanged
|
|
2086
|
+
block cost to unrelated siblings. Null marginals have no monetary edge.
|
|
2087
|
+
|
|
2088
|
+
Only reviewed actions are added. Frozen crossing costs have already been
|
|
2089
|
+
settled by the full planner, so closing these edges never reprices them.
|
|
2090
|
+
"""
|
|
2091
|
+
final = dict(current_events)
|
|
2092
|
+
for action in reviewed_actions:
|
|
2093
|
+
if action.disposition == "tombstone":
|
|
2094
|
+
final.pop(action.event_id, None)
|
|
2095
|
+
else:
|
|
2096
|
+
final[action.event_id] = {"payload": action.payload or {}}
|
|
2097
|
+
|
|
2098
|
+
def monetary_position(record):
|
|
2099
|
+
payload = (record or {}).get("payload") or {}
|
|
2100
|
+
if payload.get("kind") != "five_hour_milestone":
|
|
2101
|
+
return None
|
|
2102
|
+
threshold = _usable_threshold(payload.get("percent_threshold"))
|
|
2103
|
+
if threshold is None:
|
|
2104
|
+
return None
|
|
2105
|
+
return (_milestone_segment_key(payload), threshold,
|
|
2106
|
+
payload.get("block_cost_usd"))
|
|
2107
|
+
|
|
2108
|
+
changed = {
|
|
2109
|
+
action.event_id for action in reviewed_actions
|
|
2110
|
+
if action.kind == "five_hour_milestone"
|
|
2111
|
+
and monetary_position(current_events.get(action.event_id))
|
|
2112
|
+
!= monetary_position(final.get(action.event_id))
|
|
2113
|
+
}
|
|
2114
|
+
if not changed:
|
|
2115
|
+
return included
|
|
2116
|
+
def occupied_thresholds(events, event_ids):
|
|
2117
|
+
occupied: dict[tuple, dict[int, set[str]]] = {}
|
|
2118
|
+
for event_id in event_ids:
|
|
2119
|
+
position = monetary_position(events.get(event_id))
|
|
2120
|
+
if position is not None:
|
|
2121
|
+
segment, threshold, _cost = position
|
|
2122
|
+
occupied.setdefault(segment, {}).setdefault(
|
|
2123
|
+
threshold, set()).add(event_id)
|
|
2124
|
+
return occupied
|
|
2125
|
+
|
|
2126
|
+
edges: dict[str, set[str]] = {}
|
|
2127
|
+
for population, other in ((final, current_events), (current_events, final)):
|
|
2128
|
+
occupied = occupied_thresholds(population, population)
|
|
2129
|
+
changed_other = occupied_thresholds(other, changed)
|
|
2130
|
+
for event_id, record in population.items():
|
|
2131
|
+
payload = record.get("payload") or {}
|
|
2132
|
+
position = monetary_position(record)
|
|
2133
|
+
if (position is None or payload.get("marginal_cost_usd") is None
|
|
2134
|
+
or not _usable_amount(payload.get("block_cost_usd"))):
|
|
2135
|
+
continue
|
|
2136
|
+
segment, threshold, _cost = position
|
|
2137
|
+
lower = [value for value in occupied[segment] if value < threshold]
|
|
2138
|
+
nearest = max(lower) if lower else None
|
|
2139
|
+
dependencies = (
|
|
2140
|
+
occupied[segment][nearest] & changed if nearest is not None
|
|
2141
|
+
else set()
|
|
2142
|
+
)
|
|
2143
|
+
for prior_threshold, prior_ids in changed_other.get(segment, {}).items():
|
|
2144
|
+
if (prior_threshold < threshold
|
|
2145
|
+
and (nearest is None or prior_threshold >= nearest)):
|
|
2146
|
+
dependencies.update(prior_ids)
|
|
2147
|
+
if dependencies:
|
|
2148
|
+
edges.setdefault(event_id, set()).update(dependencies)
|
|
2149
|
+
for dependency_id in dependencies:
|
|
2150
|
+
edges.setdefault(dependency_id, set()).add(event_id)
|
|
2151
|
+
# Retained rows participate as dependency nodes too, although there is no
|
|
2152
|
+
# reviewed action to emit for them. Visit each monetary edge once rather
|
|
2153
|
+
# than rescanning the full graph for every later threshold in a chain.
|
|
2154
|
+
required_ids = {action.event_id for action in included}
|
|
2155
|
+
pending = list(required_ids)
|
|
2156
|
+
while pending:
|
|
2157
|
+
event_id = pending.pop()
|
|
2158
|
+
added = edges.get(event_id, set()) - required_ids
|
|
2159
|
+
required_ids.update(added)
|
|
2160
|
+
pending.extend(added)
|
|
2161
|
+
return [action for action in reviewed_actions
|
|
2162
|
+
if action.event_id in required_ids]
|
|
2163
|
+
|
|
2164
|
+
|
|
2165
|
+
def _causal_snapshot_closure(actions, reviewed_by_id, current_events):
|
|
2166
|
+
"""Close explicit references, keeping a selected dependency if needed."""
|
|
2167
|
+
action_by_id = {action.event_id: action for action in actions}
|
|
2168
|
+
dependency_tombstones = set()
|
|
2169
|
+
for action in actions:
|
|
2170
|
+
if (action.disposition == "tombstone"
|
|
2171
|
+
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2172
|
+
continue
|
|
2173
|
+
for field, expected_kind in _milestone_dependencies(action.kind).items():
|
|
2174
|
+
reference = (action.payload or {}).get(field)
|
|
2175
|
+
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2176
|
+
continue
|
|
2177
|
+
dependency_action = action_by_id.get(reference)
|
|
2178
|
+
if (dependency_action is not None
|
|
2179
|
+
and dependency_action.disposition == "tombstone"):
|
|
2180
|
+
current = current_events.get(reference)
|
|
2181
|
+
if ((current or {}).get("payload") or {}).get("kind") \
|
|
2182
|
+
!= expected_kind:
|
|
2183
|
+
raise RederiveConflict(
|
|
2184
|
+
"causal milestone dependency cannot be retained: "
|
|
2185
|
+
+ action.event_id)
|
|
2186
|
+
dependency_tombstones.add(reference)
|
|
2187
|
+
actions = [action for action in actions
|
|
2188
|
+
if action.event_id not in dependency_tombstones]
|
|
2189
|
+
target_events = dict(current_events)
|
|
2190
|
+
for action in actions:
|
|
2191
|
+
if action.disposition == "tombstone":
|
|
2192
|
+
target_events.pop(action.event_id, None)
|
|
2193
|
+
else:
|
|
2194
|
+
target_events[action.event_id] = {"payload": action.payload or {}}
|
|
2195
|
+
# Appending to this list intentionally visits newly required dependencies.
|
|
2196
|
+
for action in actions:
|
|
2197
|
+
if (action.disposition == "tombstone"
|
|
2198
|
+
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2199
|
+
continue
|
|
2200
|
+
for field, expected_kind in _milestone_dependencies(action.kind).items():
|
|
2201
|
+
reference = (action.payload or {}).get(field)
|
|
2202
|
+
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2203
|
+
continue
|
|
2204
|
+
target = target_events.get(reference)
|
|
2205
|
+
if ((target or {}).get("payload") or {}).get("kind") == expected_kind:
|
|
2206
|
+
continue
|
|
2207
|
+
dependency = reviewed_by_id.get(reference)
|
|
2208
|
+
if (dependency is None or dependency.disposition == "tombstone"
|
|
2209
|
+
or dependency.kind != expected_kind
|
|
2210
|
+
or (dependency.payload or {}).get("kind") != expected_kind):
|
|
2211
|
+
raise RederiveConflict(
|
|
2212
|
+
"causal milestone dependency is missing: " + action.event_id)
|
|
2213
|
+
actions.append(dependency)
|
|
2214
|
+
target_events[reference] = {"payload": dependency.payload or {}}
|
|
2215
|
+
return actions
|
|
2216
|
+
|
|
2217
|
+
|
|
1926
2218
|
def causal_delta_plan(
|
|
1927
2219
|
baseline: RederivePlan, reviewed: RederivePlan, *,
|
|
1928
2220
|
current_events: Mapping[str, Mapping],
|
|
@@ -1939,8 +2231,8 @@ def causal_delta_plan(
|
|
|
1939
2231
|
#875 (spec §4.5): a five-hour block whose close R replaces, by §4.1's
|
|
1940
2232
|
money rule, is applied whole whenever the subset changes one of its
|
|
1941
2233
|
milestones, and a block R holds frozen never is; see
|
|
1942
|
-
:func:`_replaced_block_closure`.
|
|
1943
|
-
|
|
2234
|
+
:func:`_replaced_block_closure`. Monetary predecessor/successor edges and
|
|
2235
|
+
snapshot references are closed with it until the final graph stabilizes.
|
|
1944
2236
|
"""
|
|
1945
2237
|
if (
|
|
1946
2238
|
baseline.journal_high_water != reviewed.journal_high_water
|
|
@@ -1971,73 +2263,18 @@ def causal_delta_plan(
|
|
|
1971
2263
|
)
|
|
1972
2264
|
)
|
|
1973
2265
|
]
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
reference = payload.get(field)
|
|
1987
|
-
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
1988
|
-
continue
|
|
1989
|
-
dependency_action = action_by_id.get(reference)
|
|
1990
|
-
if (dependency_action is not None
|
|
1991
|
-
and dependency_action.disposition == "tombstone"):
|
|
1992
|
-
current = current_events.get(reference)
|
|
1993
|
-
if ((current or {}).get("payload") or {}).get("kind") \
|
|
1994
|
-
!= expected_kind:
|
|
1995
|
-
raise RederiveConflict(
|
|
1996
|
-
"causal milestone dependency cannot be retained: "
|
|
1997
|
-
+ action.event_id
|
|
1998
|
-
)
|
|
1999
|
-
dependency_tombstones.add(reference)
|
|
2000
|
-
if dependency_tombstones:
|
|
2001
|
-
actions = [
|
|
2002
|
-
action for action in actions
|
|
2003
|
-
if action.event_id not in dependency_tombstones
|
|
2004
|
-
]
|
|
2005
|
-
|
|
2006
|
-
target_events = dict(current_events)
|
|
2007
|
-
for action in actions:
|
|
2008
|
-
if action.disposition == "tombstone":
|
|
2009
|
-
target_events.pop(action.event_id, None)
|
|
2010
|
-
else:
|
|
2011
|
-
target_events[action.event_id] = {"payload": action.payload or {}}
|
|
2012
|
-
for action in actions:
|
|
2013
|
-
if (action.disposition == "tombstone"
|
|
2014
|
-
or action.kind not in _SNAPSHOT_DEPENDENT_KINDS):
|
|
2015
|
-
continue
|
|
2016
|
-
payload = action.payload or {}
|
|
2017
|
-
for field, expected_kind in _milestone_dependencies(
|
|
2018
|
-
action.kind
|
|
2019
|
-
).items():
|
|
2020
|
-
reference = payload.get(field)
|
|
2021
|
-
if reference in _EMPTY_REFERENCE_SENTINELS:
|
|
2022
|
-
continue
|
|
2023
|
-
target = target_events.get(reference)
|
|
2024
|
-
if ((target or {}).get("payload") or {}).get("kind") \
|
|
2025
|
-
!= expected_kind:
|
|
2026
|
-
dependency = reviewed_by_id.get(reference)
|
|
2027
|
-
if (
|
|
2028
|
-
dependency is None
|
|
2029
|
-
or dependency.disposition == "tombstone"
|
|
2030
|
-
or dependency.kind != expected_kind
|
|
2031
|
-
or (dependency.payload or {}).get("kind") != expected_kind
|
|
2032
|
-
):
|
|
2033
|
-
raise RederiveConflict(
|
|
2034
|
-
"causal milestone dependency is missing: "
|
|
2035
|
-
+ action.event_id
|
|
2036
|
-
)
|
|
2037
|
-
actions.append(dependency)
|
|
2038
|
-
target_events[reference] = {
|
|
2039
|
-
"payload": dependency.payload or {},
|
|
2040
|
-
}
|
|
2266
|
+
# Close block, monetary and explicit-reference edges to a fixed point.
|
|
2267
|
+
# A dependency introduced by one phase must be seen by every other phase.
|
|
2268
|
+
while True:
|
|
2269
|
+
previous_ids = {action.event_id for action in actions}
|
|
2270
|
+
actions = _replaced_block_closure(
|
|
2271
|
+
actions, reviewed.actions, current_events)
|
|
2272
|
+
actions = _monetary_chain_closure(
|
|
2273
|
+
actions, reviewed.actions, current_events)
|
|
2274
|
+
actions = _causal_snapshot_closure(
|
|
2275
|
+
actions, reviewed_by_id, current_events)
|
|
2276
|
+
if {action.event_id for action in actions} == previous_ids:
|
|
2277
|
+
break
|
|
2041
2278
|
included_ids = {action.event_id for action in actions}
|
|
2042
2279
|
actions = tuple(
|
|
2043
2280
|
action for action in reviewed.actions
|
|
@@ -84,7 +84,7 @@ SIDECAR_NAME = ".segment-summaries"
|
|
|
84
84
|
#: Bumped by hand when a summary's FIELD SET or the meaning of one of its
|
|
85
85
|
#: numbers changes. It is only one of the three inputs to `summary_version`; the
|
|
86
86
|
#: other two are derived, for the reason that function's docstring gives.
|
|
87
|
-
SUMMARY_SHAPE_VERSION =
|
|
87
|
+
SUMMARY_SHAPE_VERSION = 4
|
|
88
88
|
|
|
89
89
|
#: The refusal reasons `summary_is_elidable` returns. They are stable, because
|
|
90
90
|
#: the rebuild record reports them (spec section 6.3, "recorded, not silent").
|
|
@@ -121,6 +121,14 @@ class SegmentSummary:
|
|
|
121
121
|
malformed: int = 0
|
|
122
122
|
#: True when the segment holds ZERO records of a retained type.
|
|
123
123
|
quota_only: bool = True
|
|
124
|
+
#: #825/#832 fix round A (Codex fix-round-1 Q4): True only when EVERY
|
|
125
|
+
#: decoded record in the summarized extent is a positively identified
|
|
126
|
+
#: Codex quota observation — no Claude, administrative, protocol or
|
|
127
|
+
#: unknown record. The one proof that lets a segment past the account-bound
|
|
128
|
+
#: cutover's frozen extents be elided: a summary describes its lines
|
|
129
|
+
#: WITHOUT admission, and `quota_only` alone still admits a stamped Claude
|
|
130
|
+
#: observation that feeds the quarantine and the derivation completion.
|
|
131
|
+
codex_quota_only: bool = False
|
|
124
132
|
#: Elements the segment contributed to the rebuild's `decoded` list —
|
|
125
133
|
#: retained records plus placeholders, and never a malformed line. `None`
|
|
126
134
|
#: means the summary predates the field and cannot be elided against.
|
|
@@ -227,7 +235,8 @@ def summary_is_elidable(
|
|
|
227
235
|
_FIELDS = (
|
|
228
236
|
"segment_name", "st_dev", "st_ino", "summarized_size",
|
|
229
237
|
"complete_line_covered_offset", "lines", "bytes", "decodes", "malformed",
|
|
230
|
-
"quota_only", "
|
|
238
|
+
"quota_only", "codex_quota_only", "decoded_entry_count",
|
|
239
|
+
"complete_prefix_sha256",
|
|
231
240
|
"last_seen_stamped",
|
|
232
241
|
"last_seen_legacy_claude_at", "last_seen_legacy_codex_at",
|
|
233
242
|
)
|
|
@@ -285,6 +294,7 @@ def to_wire(summary) -> dict:
|
|
|
285
294
|
"decodes": int(summary.decodes),
|
|
286
295
|
"malformed": int(summary.malformed),
|
|
287
296
|
"quota_only": bool(summary.quota_only),
|
|
297
|
+
"codex_quota_only": bool(summary.codex_quota_only),
|
|
288
298
|
"decoded_entry_count": (
|
|
289
299
|
None if summary.decoded_entry_count is None
|
|
290
300
|
else int(summary.decoded_entry_count)),
|
|
@@ -326,6 +336,7 @@ def from_wire(item) -> SegmentSummary:
|
|
|
326
336
|
decodes=int(item["decodes"]),
|
|
327
337
|
malformed=int(item["malformed"]),
|
|
328
338
|
quota_only=bool(item["quota_only"]),
|
|
339
|
+
codex_quota_only=bool(item["codex_quota_only"]),
|
|
329
340
|
decoded_entry_count=None if count is None else int(count),
|
|
330
341
|
complete_prefix_sha256=(
|
|
331
342
|
None if item["complete_prefix_sha256"] is None
|
|
@@ -427,6 +438,28 @@ def read_sidecar(path):
|
|
|
427
438
|
return None
|
|
428
439
|
|
|
429
440
|
|
|
441
|
+
def admission_permits_elision(boundary, name, pinned_raw_extent,
|
|
442
|
+
summary) -> bool:
|
|
443
|
+
"""Whether the account-bound admission fence lets segment ``name`` be
|
|
444
|
+
elided at all (#825/#832 fix round A, Codex fix-round-1 Q4). The rebuild
|
|
445
|
+
and the prefix validation share it; every other elision condition is
|
|
446
|
+
still `summary_is_elidable`'s.
|
|
447
|
+
|
|
448
|
+
A summary describes its lines without admission. A wholly historical
|
|
449
|
+
extent (inside the cutover's frozen extent for that segment) needs no
|
|
450
|
+
admission. Past it, only a summary proving every decoded record a Codex
|
|
451
|
+
quota observation — always admitted, never a quarantine or completion
|
|
452
|
+
input — may stand in for reading the segment; anything unproven (no
|
|
453
|
+
summary, an older shape, a mixed segment) is scanned.
|
|
454
|
+
"""
|
|
455
|
+
if boundary is None:
|
|
456
|
+
return True
|
|
457
|
+
extent = boundary.frozen_extents.get(name)
|
|
458
|
+
if extent is not None and int(pinned_raw_extent) <= int(extent):
|
|
459
|
+
return True
|
|
460
|
+
return summary is not None and bool(summary.codex_quota_only)
|
|
461
|
+
|
|
462
|
+
|
|
430
463
|
def with_identity(summary, *, st_dev, st_ino) -> SegmentSummary:
|
|
431
464
|
"""``summary`` restamped with the inode identity of the file just read."""
|
|
432
465
|
return replace(summary, st_dev=int(st_dev), st_ino=int(st_ino))
|
|
@@ -1315,6 +1315,15 @@ class CodexAccountingCacheResult:
|
|
|
1315
1315
|
#: the generation this capture builds and admits a retained generation
|
|
1316
1316
|
#: only while its stamp equals `codex_accounting_consumed_provenance()`.
|
|
1317
1317
|
provenance_token: int | None = None
|
|
1318
|
+
#: #872: the NAME of the population `changed_old` / `changed_new` are
|
|
1319
|
+
#: relative to — the prior state's `population_signature` and provenance
|
|
1320
|
+
#: token, read before this call mutated the state. `None` on every COLD
|
|
1321
|
+
#: result: a cold delta replaces the whole population relative to nothing
|
|
1322
|
+
#: a consumer can name (an empty state, a reset, an overflow clear, or a
|
|
1323
|
+
#: semantic/start/end/sequence/overflow cold trigger), so a consumer must
|
|
1324
|
+
#: derive from the full population instead of advancing retained state.
|
|
1325
|
+
base_population_signature: tuple | None = None
|
|
1326
|
+
base_provenance_token: int | None = None
|
|
1318
1327
|
|
|
1319
1328
|
|
|
1320
1329
|
_CODEX_ACCOUNTING_CACHE_STATE: dict[str, object] = _ObservedSnapshotDict()
|
|
@@ -1551,6 +1560,14 @@ def build_cached_codex_accounting(
|
|
|
1551
1560
|
attribution_revision = _codex_window_attribution_revision(cache_conn)
|
|
1552
1561
|
state = _CODEX_ACCOUNTING_CACHE_STATE
|
|
1553
1562
|
prior_cached_entries = tuple(state.get("entries", ())) if state else ()
|
|
1563
|
+
# #872: the delta's base, read before any branch below mutates the state.
|
|
1564
|
+
prior_signature = state.get("population_signature") if state else None
|
|
1565
|
+
if not isinstance(prior_signature, tuple):
|
|
1566
|
+
prior_signature = None
|
|
1567
|
+
prior_token = (
|
|
1568
|
+
state.get(_CODEX_ACCOUNTING_PROVENANCE_STATE_KEY) if state else None)
|
|
1569
|
+
if not isinstance(prior_token, int):
|
|
1570
|
+
prior_token = None
|
|
1554
1571
|
|
|
1555
1572
|
def signature(generation: int) -> tuple | None:
|
|
1556
1573
|
"""The exact identity of the population this call is returning.
|
|
@@ -1683,6 +1700,8 @@ def build_cached_codex_accounting(
|
|
|
1683
1700
|
population_signature=population_signature,
|
|
1684
1701
|
provenance_token=_consume_codex_accounting_provenance(
|
|
1685
1702
|
provenance_token),
|
|
1703
|
+
base_population_signature=prior_signature,
|
|
1704
|
+
base_provenance_token=prior_token,
|
|
1686
1705
|
)
|
|
1687
1706
|
|
|
1688
1707
|
dirty_paths = tuple(sorted(dirty))
|
|
@@ -1736,6 +1755,8 @@ def build_cached_codex_accounting(
|
|
|
1736
1755
|
entries, dirty_paths, dirty_accounts, False, changed_old, changed_new,
|
|
1737
1756
|
population_signature=population_signature,
|
|
1738
1757
|
provenance_token=_consume_codex_accounting_provenance(provenance_token),
|
|
1758
|
+
base_population_signature=prior_signature,
|
|
1759
|
+
base_provenance_token=prior_token,
|
|
1739
1760
|
)
|
|
1740
1761
|
|
|
1741
1762
|
|