cctally 1.113.0 → 1.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +32 -8
  4. package/bin/_cctally_cache.py +6035 -722
  5. package/bin/_cctally_completion.py +1287 -0
  6. package/bin/_cctally_config.py +37 -0
  7. package/bin/_cctally_core.py +400 -2
  8. package/bin/_cctally_dashboard.py +98 -20
  9. package/bin/_cctally_dashboard_conversation.py +292 -23
  10. package/bin/_cctally_dashboard_envelope.py +16 -3
  11. package/bin/_cctally_dashboard_perf.py +128 -9
  12. package/bin/_cctally_dashboard_sources.py +285 -213
  13. package/bin/_cctally_db.py +1177 -23
  14. package/bin/_cctally_diagnosis_sources.py +22 -4
  15. package/bin/_cctally_doctor.py +572 -45
  16. package/bin/_cctally_five_hour.py +84 -33
  17. package/bin/_cctally_forecast.py +108 -15
  18. package/bin/_cctally_journal.py +3950 -116
  19. package/bin/_cctally_journal_repair.py +101 -12
  20. package/bin/_cctally_milestone_history.py +82 -9
  21. package/bin/_cctally_parser.py +42 -0
  22. package/bin/_cctally_percent_breakdown.py +3 -2
  23. package/bin/_cctally_project.py +8 -2
  24. package/bin/_cctally_quarantine.py +652 -0
  25. package/bin/_cctally_quota.py +227 -37
  26. package/bin/_cctally_quota_model.py +360 -83
  27. package/bin/_cctally_record.py +691 -149
  28. package/bin/_cctally_rederive.py +320 -41
  29. package/bin/_cctally_refresh.py +84 -13
  30. package/bin/_cctally_source_analytics.py +83 -17
  31. package/bin/_cctally_statusline.py +6154 -517
  32. package/bin/_cctally_store.py +367 -53
  33. package/bin/_cctally_transcript.py +8 -7
  34. package/bin/_cctally_tui.py +61 -56
  35. package/bin/_cctally_weekrefs.py +128 -21
  36. package/bin/_lib_account_binding.py +152 -0
  37. package/bin/_lib_account_protocol.py +386 -0
  38. package/bin/_lib_aggregators.py +4 -0
  39. package/bin/_lib_cache_coverage.py +13 -1
  40. package/bin/_lib_cache_report.py +35 -6
  41. package/bin/_lib_cache_writer_lock.py +19 -3
  42. package/bin/_lib_clone_copy.py +142 -0
  43. package/bin/_lib_codex_conversation.py +32 -0
  44. package/bin/_lib_codex_conversation_query.py +297 -49
  45. package/bin/_lib_codex_conversation_watch.py +59 -14
  46. package/bin/_lib_codex_drain_admission.py +214 -0
  47. package/bin/_lib_codex_metadata.py +18 -1
  48. package/bin/_lib_codex_publication_barrier.py +277 -0
  49. package/bin/_lib_codex_rollout_container.py +502 -0
  50. package/bin/_lib_conversation_dispatch.py +3 -0
  51. package/bin/_lib_conversation_query.py +106 -59
  52. package/bin/_lib_conversation_retention.py +1605 -640
  53. package/bin/_lib_cost_provenance.py +53 -2
  54. package/bin/_lib_detector_access_registry.py +3817 -0
  55. package/bin/_lib_doctor.py +714 -69
  56. package/bin/_lib_five_hour.py +126 -0
  57. package/bin/_lib_fmt.py +11 -0
  58. package/bin/_lib_ingest_frontier.py +414 -20
  59. package/bin/_lib_journal.py +141 -10
  60. package/bin/_lib_jsonl.py +5 -4
  61. package/bin/_lib_legacy_spool.py +126 -0
  62. package/bin/_lib_legacy_writer_evidence.py +136 -0
  63. package/bin/_lib_merge_gate.py +231 -86
  64. package/bin/_lib_meter_rate_change.py +10 -0
  65. package/bin/_lib_outcome_quarantine.py +690 -0
  66. package/bin/_lib_perf.py +27 -1
  67. package/bin/_lib_pricing.py +199 -9
  68. package/bin/_lib_pricing_check.py +20 -4
  69. package/bin/_lib_quarantine.py +154 -0
  70. package/bin/_lib_quota_calibration.py +53 -8
  71. package/bin/_lib_quota_copy.py +2 -1
  72. package/bin/_lib_quota_model.py +288 -46
  73. package/bin/_lib_reclaim_planner.py +792 -0
  74. package/bin/_lib_rederive.py +319 -82
  75. package/bin/_lib_segment_summary.py +35 -2
  76. package/bin/_lib_snapshot_cache.py +21 -0
  77. package/bin/_lib_sqlite_close.py +81 -0
  78. package/bin/_lib_statusline_candidates.py +292 -23
  79. package/bin/_lib_subscription_weeks.py +100 -17
  80. package/bin/_lib_tick_stats.py +217 -17
  81. package/bin/_lib_wal_checkpoint.py +1077 -0
  82. package/bin/_lib_write_budget.py +483 -0
  83. package/bin/_lib_write_io.py +703 -0
  84. package/bin/cctally +31 -1
  85. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  86. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  87. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  88. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  89. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  90. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  91. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  92. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  93. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  94. package/dashboard/static/dashboard.html +2 -2
  95. package/package.json +19 -1
  96. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  97. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -1,4 +1,5 @@
1
- """#313 P3: conversation-transcript retention prune kernel.
1
+ """#313 P3 + #901: conversation-transcript retention — paced expiry deletion
2
+ and paced reclaim.
2
3
 
3
4
  Prunes ONLY the re-derivable transcript rows:
4
5
  * Claude: ``conversation_messages`` + their ``conversation_file_touches`` /
@@ -19,214 +20,863 @@ least one dated row and NO row at/after the cutoff. Rows whose timestamps are
19
20
  entirely NULL are treated conservatively and never pruned in isolation (F12).
20
21
  NULL identity (``session_id`` / ``conversation_key``) falls back to grouping by
21
22
  ``source_path`` so malformed rows stay bounded rather than orphaned-unbounded
22
- (F12).
23
-
24
- The FTS5 indexes over ``conversation_messages`` (``conversation_fts``) and
25
- ``conversation_ai_titles`` (``conversation_title_fts``) are external-content and
26
- maintained by AFTER-DELETE triggers, which are logically correct on subset
27
- deletes. Whole groups are deleted so those triggers keep the index consistent.
28
- The kernel normally runs inside the caller's open transaction. The orchestrator
29
- supplies a post-group boundary that commits each whole conversation separately
30
- while retaining every flock for the full pass (#315), bounding WAL growth.
23
+ (F12). The FTS5 indexes are external-content and maintained by AFTER-DELETE
24
+ triggers, which stay consistent because whole groups are deleted.
25
+
26
+ #901 (spec §5.4; operator decisions Q2, Q4, Q5, Q7) puts every maintenance
27
+ write on ONE durable allowance:
28
+
29
+ * 4 MiB/min with at most one minute (4 MiB) of credit, accrued by wall clock
30
+ from the durable ``as_of`` and capped, so a restart never refills credit and
31
+ never bypasses the cadence. The balance and the 25-bucket hourly charge
32
+ ledger live in the extended #780 record (``RECLAIM_PENDING_KEY``).
33
+ * An OPERATION is one expired group's deletion or one reclaim chunk. It starts
34
+ only from a non-negative balance and is charged EXACTLY its reservation,
35
+ written in the operation's own ``BEGIN IMMEDIATE`` transaction together with
36
+ its mutation: a crash before COMMIT leaves neither, after it both. Overshoot
37
+ becomes debt and nothing runs until it is repaid. There is no reconciliation
38
+ and no deferred-copy record.
39
+ * Reclaim never checkpoints: its connection runs ``wal_autocheckpoint = 0`` with
40
+ SQLite's no-checkpoint-on-close setting on (plus ``cache_spill = OFF`` and an
41
+ in-memory temp store), and when those settings cannot be applied the attempt
42
+ runs nothing and records why. It starts only when free pages exceed 2 GiB
43
+ AND 20% of the file, stops at 1 GiB OR 10%, and runs at most once a minute
44
+ under the 2 s deadline. Each chunk (Q9) is planned inside its own
45
+ ``BEGIN IMMEDIATE`` by `_lib_reclaim_planner` in a helper subprocess, runs at
46
+ most 16 ``incremental_vacuum(1)`` steps, and is charged
47
+ (|ID| + UNID + 1) x (2P + 24) + F_r from that plan; a planner refusal rolls
48
+ back with no vacuum and no charge and is reported in memory only.
49
+ * Deletion keeps whole-group atomicity and SQLite's automatic PASSIVE checkpoint
50
+ at COMMIT, on a per-visit connection with a 256 MiB cache and an in-memory
51
+ temp store, so a group is deleted spill-free; above 60,000 rows the cache cap
52
+ stays and the statement journal uses the file temp store (Q5). Its charge
53
+ (Q8) is F + A x rows + B x M_cap, M_cap the pointer-map worst case of the
54
+ file's geometry read inside the transaction. A visit stops between groups
55
+ when the balance is negative and persists a continuation cutoff; the daily
56
+ stamp is written only when both providers finish.
57
+ * The #780 record is one fixed-size cell (`RECLAIM_RECORD_LENGTH`): every
58
+ writer stores exactly L bytes and none deletes the row, so a chunk's state
59
+ write overwrites it in place and dirties one page.
31
60
  """
32
61
  from __future__ import annotations
33
62
 
63
+ import dataclasses
34
64
  import datetime as dt
35
65
  import fcntl
36
66
  import json
37
67
  import os
38
68
  import sqlite3
69
+ import subprocess
70
+ import sys
39
71
  import time
40
72
  from collections.abc import Callable
41
73
  from dataclasses import dataclass
42
74
 
43
75
  import _cctally_core
76
+ import _lib_reclaim_planner as _planner
77
+ import _lib_sqlite_close
78
+ import _lib_write_budget as _budget
79
+ import _lib_write_io
44
80
 
45
81
  UTC = dt.timezone.utc
82
+ KiB = 1024
83
+ MiB = 1024 * KiB
84
+ GiB = 1024 * MiB
46
85
 
47
86
  # cache_meta throttle key (framework-untracked KV — NO schema migration, F7).
48
87
  _RETENTION_LAST_PRUNE_KEY = "conversation_retention_last_prune_at"
49
88
  _RETENTION_THROTTLE_SECONDS = 24 * 60 * 60
50
89
 
51
- # --------------------------------------------------------------------------
52
- # #780 — bounded, resumable reclaim
53
- # --------------------------------------------------------------------------
54
-
55
- #: Durable reclaim state (framework-untracked KV, same class as the throttle
56
- #: key above — a value shape, not a schema change).
90
+ #: The extended #780 record: balance, ledger, episode and progress. The key is
91
+ #: unchanged so an older binary still reads its legacy fields.
57
92
  RECLAIM_PENDING_KEY = "conversation_retention_reclaim_pending"
58
-
59
- #: One reclaim pass gets this much wall clock. The pass runs under the
60
- #: maintenance flock and BOTH provider flocks, so an unbounded run holds all
61
- #: three: the previous implementation drove the freelist to completion in
62
- #: 4096-page chunks with no budget at all, and on a multi-GiB store that is
63
- #: minutes of held locks against a live reader surface.
64
- #:
65
- #: THE DEADLINE IS CHECKED BETWEEN CHUNKS, so a pass can exceed it by at most
66
- #: one chunk, and that bound is stated rather than hidden because
67
- #: `PRAGMA incremental_vacuum(n)` cannot be preempted. Chunk size is the only
68
- #: control, and the cost of a chunk is dominated by CACHE STATE, not by `n`.
69
- #:
70
- #: Measured on a copy-on-write clone of the production store, cold: 1 page
71
- #: 0.006 s, 8 pages 0.001 s, 64 pages 0.009 s, 512 pages 7.754 s. An earlier
72
- #: version of this comment read that series as per-page cost rising with chunk
73
- #: size and concluded that "no estimator fixes this". The Tranche 3 review
74
- #: disproved it and the correction matters, because the false version tells
75
- #: the next maintainer not to bother measuring. Two facts settle it. Warm
76
- #: passes on the SAME store sustain about 2,300 pages a second at 1,024 and
77
- #: 2,048-page chunks, so a large chunk is not intrinsically expensive per page.
78
- #: And the 8-page chunk being SIX TIMES cheaper than the 1-page chunk is not a
79
- #: size effect at all — it is the second reading of a cold ramp warming up.
80
- #: The 512-page chunk is the fourth reading of that ramp, and it is expensive
81
- #: because it is the first one to reach a genuinely cold region of a multi-GiB
82
- #: file, not because it asked for 512 pages.
83
- #:
84
- #: So the real hazard is that the ramp doubles into cold territory before it
85
- #: has measured a slow chunk there — an estimator built from warm readings
86
- #: cannot see the first-touch cost of pages nobody has read yet. The growth cap
87
- #: below is the control that matters for exactly that reason, and a plausible
88
- #: refinement, not taken here because it is unmeasured, is to require two or
89
- #: three consecutive chunks at the current size before permitting a doubling,
90
- #: so a size increase always rests on more than one observation.
91
- #:
92
- #: Observed worst case on that store: 9.9 s for the first, cold pass, then
93
- #: 2.05-2.11 s for every warm pass. That is the overshoot this design accepts,
94
- #: and it is bounded and reported (`deadline_hit`) rather than unbounded: the
95
- #: implementation it replaces held the maintenance flock and both provider
96
- #: flocks for the whole drain, which on the same store was part of a 25-minute
97
- #: rebuild.
93
+ RETENTION_POLICY_VERSION = 1
94
+ _LEGACY_RECORD_KEYS = ("freelist_count", "unreclaimed_bytes", "attempted_at",
95
+ "made_progress", "deadline_hit")
96
+
97
+ # ── Reservation constants: CANDIDATES (spec §5.4, Q8 and Q9) ──────────────
98
+ # Fixed by the §6.3 R-cov calibration: each calibrated part must be at least
99
+ # 1.25 x its measured maximum, rounded up to a KiB. A candidate that falls
100
+ # short is raised, never left below coverage. Calibration edits THIS block
101
+ # only. The analytical page terms (B's 2P + 24 per pointer-map page, and a
102
+ # reclaim chunk's planned pages) are computed, never calibrated downwards.
103
+ #: The formula version every operation record carries; a record with
104
+ #: another version is not recomputed (`recompute_charge`).
105
+ RESERVATION_VERSION = 2
106
+ #: F: a deletion's fixed part per operation (the WAL header is inside it).
107
+ DELETION_FIXED_BYTES = 512 * KiB
108
+ #: A: per deleted row, and above `DELETION_FALLBACK_ROW_THRESHOLD` (Q5).
109
+ DELETION_PER_ROW_BYTES = 15 * KiB
110
+ DELETION_PER_ROW_FALLBACK_BYTES = 50 * KiB
111
+ #: B = ceil(5/4 x (2P + 24) / 1 KiB) KiB per pointer-map page of M_cap, from
112
+ #: the store's own page size (11 KiB at 4 KiB pages).
113
+ POINTER_MAP_FACTOR_NUMERATOR = 5
114
+ POINTER_MAP_FACTOR_DENOMINATOR = 4
115
+ #: F_r: a reclaim chunk's fixed part (WAL header and framing; Q9). The page
116
+ #: term (|ID| + UNID + 1) x (2P + 24) comes from the chunk's own plan.
117
+ RECLAIM_FIXED_BYTES = 64 * KiB
118
+ # ──────────────────────────────────────────────────────────────────────────
119
+
120
+ #: I4's deletion write bounds (spec §3; limits on the writes themselves,
121
+ #: never calibrated). N_max's growth term is built from the first two.
122
+ I4_DELETION_FIXED_BYTES = 512 * KiB
123
+ I4_DELETION_PER_ROW_BYTES = 12 * KiB
124
+ I4_FALLBACK_PER_ROW_BYTES = 40 * KiB
125
+
126
+ #: Q5: a group above this many rows keeps the cache cap but uses the file
127
+ #: temp store for its statement journal, bounded at 40 KiB per deleted row.
128
+ DELETION_FALLBACK_ROW_THRESHOLD = 60_000
129
+ #: `PRAGMA cache_size = -262144`: a 256 MiB cap, used only as pages dirty.
130
+ DELETION_CACHE_SIZE_KIB = 262_144
131
+
132
+ RECLAIM_START_BYTES = 2 * GiB
133
+ RECLAIM_START_RATIO = 0.20
134
+ RECLAIM_STOP_BYTES = 1 * GiB
135
+ RECLAIM_STOP_RATIO = 0.10
136
+ RECLAIM_MAX_CHUNK_PAGES = 16
137
+ RECLAIM_ATTEMPT_INTERVAL_SECONDS = 60
138
+ #: The secondary deadline between chunks; a chunk itself is not preemptible.
98
139
  RECLAIM_DEADLINE_SECONDS = 2.0
99
-
100
- #: Chunk sizing. Start small so the FIRST chunk cannot overshoot the deadline
101
- #: on a cold multi-GiB file, then adapt upward from measured throughput —
102
- #: DOUBLING at most, and from the SLOWEST rate this pass has seen rather than
103
- #: the most recent one. Neither is a cure — see the cold-cache first-touch cost
104
- #: recorded on the deadline above — but together they bound the growth to one
105
- #: doubling per chunk, so the pass takes many measured steps and the single
106
- #: chunk that can overshoot is at most twice the last chunk that fit inside the
107
- #: budget.
108
- RECLAIM_INITIAL_PAGES = 64
109
- RECLAIM_MAX_PAGES = 2048
110
- RECLAIM_CHUNK_GROWTH_FACTOR = 2
111
-
112
- #: Backlog bounds. "Unbounded growth is not accepted" needs numbers, so these
113
- #: are them, and each has a distinct consequence.
114
- #: * above ESCALATION, reclaim stops waiting for the daily throttle and
115
- #: continues on the next cycle;
116
- #: * below MIN_PAGES_PER_SECOND, the pass is recorded as making no progress
117
- #: and stays pending — it is never declared complete;
118
- #: * above CEILING, doctor raises a FAIL and a new rebuild is refused,
119
- #: because a rebuild adds staging churn to a store already failing to drain.
120
- #:
121
- #: THE CEILING IS SIZED FROM A MEASUREMENT, not chosen. One full rebuild of a
122
- #: copy-on-write clone of the 9.2 GB production store left 1,398,528 free pages
123
- #: — 5.33 GiB, 37% of the file — because reclaim is now budgeted and the
124
- #: remainder is deferred. A ceiling below that would FAIL doctor and refuse the
125
- #: next rebuild after every ordinary production-scale rebuild, which is a false
126
- #: alarm rather than an enforcement mechanism. 16 GiB is three times the
127
- #: measured single-rebuild leftover.
128
- #:
129
- #: The escalation threshold is what actually drains it. Measured steady rate on
130
- #: the same clone: about 2,300 pages a second, so roughly 4,600 pages a 2 s
131
- #: pass, so about 300 passes for that backlog. At the daily throttle that is
132
- #: 300 days; at the escalated per-cycle cadence it is under an hour. That
133
- #: ratio is the reason escalation exists.
134
- RECLAIM_ESCALATION_BYTES = 256 * 1024 * 1024
135
- RECLAIM_MIN_PAGES_PER_SECOND = 1.0
136
- RECLAIM_CEILING_BYTES = 16 * 1024 * 1024 * 1024
137
-
138
-
139
- @dataclass(frozen=True)
140
- class ReclaimOutcome:
141
- """What one bounded reclaim pass observed. Never a claim about the file."""
142
-
143
- freelist_before: int = 0
144
- freelist_after: int = 0
145
- page_size: int = 0
146
- pages_reclaimed: int = 0
147
- duration_s: float = 0.0
148
- deadline_hit: bool = False
149
-
150
- @property
151
- def unreclaimed_bytes(self) -> int:
152
- return max(0, self.freelist_after) * max(0, self.page_size)
153
-
154
- @property
155
- def made_progress(self) -> bool:
156
- if self.pages_reclaimed <= 0:
157
- return False
158
- if self.duration_s <= 0:
159
- return True
160
- return (self.pages_reclaimed / self.duration_s) >= (
161
- RECLAIM_MIN_PAGES_PER_SECOND)
162
-
163
- @property
164
- def freelist_drained(self) -> bool:
165
- return self.freelist_after <= 0
140
+ #: Doctor FAIL and rebuild refusal, judged on the OBSERVED freelist; reaching
141
+ #: it never accelerates automatic writes.
142
+ RECLAIM_CEILING_BYTES = 16 * GiB
143
+ #: A chunk's whole reservation target (Q9): the plan grows one step at a
144
+ #: time while it stays at most this; a single step over it runs alone.
145
+ RECLAIM_CHUNK_BUDGET_BYTES = 4 * MiB
146
+ #: Every typed reason a reclaim attempt can skip with (spec §5.4 "Refusal"):
147
+ #: the connection's settings, the record's precondition, the runtime SQLite,
148
+ #: the planner's own refusals and its helper, and an unexpected SQLite error.
149
+ RECLAIM_SKIP_REASONS = (
150
+ "no_checkpoint_on_close_unavailable", "connection_settings_unavailable",
151
+ "sqlite_error", "state_record_unnormalized", "state_record_oversize",
152
+ "helper_failed", "helper_timeout") + _planner.REFUSAL_REASONS
153
+ #: The planner helper: a fresh interpreter (never a fork of this process),
154
+ #: so no raw descriptor of the store is ever opened here. It imports only
155
+ #: the standard library.
156
+ _PLANNER_COMMAND = [sys.executable,
157
+ os.path.join(os.path.dirname(os.path.abspath(
158
+ _planner.__file__)), "_lib_reclaim_planner.py")]
159
+ #: The helper's time budget when a chunk runs outside an attempt; inside
160
+ #: one it is the rest of the attempt's 2 s secondary deadline.
161
+ _PLANNER_DEADLINE_SECONDS = 1.5
162
+ _PLANNER_MIN_DEADLINE_SECONDS = 0.05
163
+ #: Interpreter start-up allowance on top of the helper's own budget.
164
+ _PLANNER_STARTUP_SLACK_SECONDS = 1.0
165
+ #: The helper's read budget in bytes (its time budget is the deadline).
166
+ _PLANNER_READ_BUDGET_BYTES = 256 * MiB
167
+
168
+ #: Mirror of `_cctally_store.STORE_POLICY["conversations"]`'s connection
169
+ #: settings; `tests/test_retention_pacing.py` pins the equality.
170
+ _CONNECTION_BUSY_TIMEOUT_MS = 15_000
171
+ _CONNECTION_SYNCHRONOUS = "NORMAL"
172
+ _CONNECTION_JOURNAL_SIZE_LIMIT = 128 * MiB
166
173
 
167
174
 
168
175
  @dataclass(frozen=True)
169
176
  class PruneStats:
170
- """Counts from one prune pass."""
177
+ """Counts from one prune pass. ``complete`` is False when a paced visit
178
+ stopped before every expired group was deleted (a continuation)."""
171
179
 
172
180
  claude_sessions: int = 0
173
181
  claude_messages: int = 0
174
182
  codex_conversations: int = 0
175
183
  codex_events: int = 0
184
+ complete: bool = True
176
185
 
177
186
  @property
178
187
  def total_rows(self) -> int:
179
188
  return self.claude_messages + self.codex_events
180
189
 
181
190
 
191
+ def _iso(value: "dt.datetime | None") -> "str | None":
192
+ return None if value is None else _budget.iso_z(value)
193
+
194
+
195
+ def _ceil_second(value: dt.datetime) -> dt.datetime:
196
+ """``value`` rounded UP to a whole UTC second (idempotent)."""
197
+ value = value.astimezone(UTC)
198
+ if value.microsecond:
199
+ value = value.replace(microsecond=0) + dt.timedelta(seconds=1)
200
+ return value
201
+
202
+
203
+ def _iso_ceil(value: "dt.datetime | None") -> "str | None":
204
+ """A pacing instant at whole-second precision, rounded UP (901-RW-001).
205
+
206
+ The record's instants are 20-character whole seconds. Truncating the
207
+ allowance's ``as_of`` let every reload of the record at the same
208
+ fractional visit instant accrue the dropped fraction again (I4), and
209
+ truncating ``next_attempt_at`` moved the reclaim cadence earlier. Rounding
210
+ up can only withhold accrual and delay a deadline."""
211
+ return None if value is None else _budget.iso_z(_ceil_second(value))
212
+
213
+
214
+ def _parse_utc(value) -> "dt.datetime | None":
215
+ if not isinstance(value, str):
216
+ return None
217
+ try:
218
+ parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
219
+ except ValueError:
220
+ return None
221
+ if parsed.tzinfo is None:
222
+ parsed = parsed.replace(tzinfo=UTC)
223
+ return parsed.astimezone(UTC)
224
+
225
+
226
+ def _int_or_none(value) -> "int | None":
227
+ if isinstance(value, bool) or not isinstance(value, int):
228
+ return None
229
+ return value
230
+
231
+
182
232
  def _cutoff_iso(cutoff_utc: dt.datetime) -> str:
183
233
  """Whole-second UTC ``...Z`` boundary for lex comparison against the stored
184
- ``...Z`` timestamps.
185
-
186
- Second-granular: the sub-second mixed-precision edge at the exact cutoff
187
- second is immaterial for a multi-day retention window — any message wrongly classified
188
- there is re-derivable from JSONL and the boundary self-corrects on the next
189
- (daily) prune.
190
- """
234
+ ``...Z`` timestamps (second-granular; immaterial for a multi-day window)."""
191
235
  return cutoff_utc.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
192
236
 
193
237
 
194
- def prune_conversation_transcripts(
195
- conn: sqlite3.Connection,
196
- *,
197
- cutoff_utc: dt.datetime,
198
- after_group: "Callable[[], None] | None" = None,
199
- ) -> PruneStats:
200
- """Prune every transcript group whose latest activity is before ``cutoff_utc``.
238
+ # ── pacing state (the extended #780 record) ───────────────────────────────
201
239
 
202
- ``after_group`` runs only after every table and FTS posting owned by one
203
- session/conversation has been deleted. Direct kernel callers leave it unset
204
- and retain their caller-managed transaction; the orchestrator uses it for
205
- #315's whole-conversation intermediate commits.
206
- """
207
- cutoff = _cutoff_iso(cutoff_utc)
208
- claude_sessions, claude_messages = _prune_claude(
209
- conn, cutoff, after_group=after_group
210
- )
211
- codex_conversations, codex_events = _prune_codex(
212
- conn, cutoff, after_group=after_group
213
- )
214
- return PruneStats(
215
- claude_sessions=claude_sessions,
216
- claude_messages=claude_messages,
217
- codex_conversations=codex_conversations,
218
- codex_events=codex_events,
240
+ @dataclass(frozen=True)
241
+ class PacingState:
242
+ balance_bytes: int
243
+ as_of: dt.datetime
244
+ ledger: tuple = ()
245
+ eligible: bool = False
246
+ next_attempt_at: "dt.datetime | None" = None
247
+ continuation_cutoff: "str | None" = None
248
+ last_failure: "dict | None" = None
249
+ progress: "dict | None" = None
250
+ op_seq: int = 0
251
+ legacy: dict = dataclasses.field(default_factory=dict)
252
+
253
+ @classmethod
254
+ def from_record(cls, record, now_utc: dt.datetime) -> "PacingState":
255
+ """A legacy or absent record reads as full credit and no charges."""
256
+ now = now_utc.astimezone(UTC)
257
+ legacy = {}
258
+ if isinstance(record, dict):
259
+ legacy = {k: record[k] for k in _LEGACY_RECORD_KEYS if k in record}
260
+ if not isinstance(record, dict) or record.get("policy_version") is None:
261
+ return cls(balance_bytes=_budget.MAINTENANCE_CREDIT_BYTES,
262
+ as_of=now, legacy=legacy)
263
+ balance = _int_or_none(record.get("balance_bytes"))
264
+ as_of = _parse_utc(record.get("as_of"))
265
+ if balance is None or as_of is None:
266
+ balance, as_of = _budget.MAINTENANCE_CREDIT_BYTES, now
267
+ failure = record.get("last_failure")
268
+ progress = record.get("progress")
269
+ cutoff = record.get("continuation_cutoff")
270
+ return cls(
271
+ balance_bytes=min(balance, _budget.MAINTENANCE_CREDIT_BYTES),
272
+ as_of=as_of,
273
+ ledger=tuple(_budget.serialize_ledger(
274
+ _budget.parse_ledger(record.get("ledger")))),
275
+ eligible=record.get("eligible") is True,
276
+ next_attempt_at=_parse_utc(record.get("next_attempt_at")),
277
+ continuation_cutoff=cutoff if isinstance(cutoff, str) else None,
278
+ last_failure=failure if isinstance(failure, dict) else None,
279
+ progress=progress if isinstance(progress, dict) else None,
280
+ op_seq=max(0, _int_or_none(record.get("op_seq")) or 0),
281
+ legacy=legacy,
282
+ )
283
+
284
+ def available(self, now_utc: dt.datetime) -> int:
285
+ elapsed = max(0.0, (now_utc.astimezone(UTC) - self.as_of).total_seconds())
286
+ accrued = self.balance_bytes + int(
287
+ _budget.MAINTENANCE_RATE_BYTES_PER_MINUTE * elapsed / 60.0)
288
+ return min(_budget.MAINTENANCE_CREDIT_BYTES, accrued)
289
+
290
+ def charge(self, reservation: int, now_utc: dt.datetime, *,
291
+ started_utc: "dt.datetime | None" = None) -> "PacingState":
292
+ """Subtract the reservation at ``now_utc`` (the accrual instant) and
293
+ add it to the bucket of the hour the operation started,
294
+ ``started_utc`` (PR-13 (a); the accrual instant when omitted)."""
295
+ now = now_utc.astimezone(UTC)
296
+ start = now if started_utc is None else started_utc.astimezone(UTC)
297
+ return dataclasses.replace(
298
+ self,
299
+ balance_bytes=self.available(now) - int(reservation),
300
+ as_of=max(now, self.as_of),
301
+ ledger=tuple(_budget.ledger_add(
302
+ list(self.ledger), at_utc=start,
303
+ charge_bytes=int(reservation))),
304
+ op_seq=self.op_seq + 1,
305
+ )
306
+
307
+ def to_record(self) -> dict:
308
+ record = dict(self.legacy)
309
+ record.update({
310
+ "policy_version": RETENTION_POLICY_VERSION,
311
+ "balance_bytes": int(self.balance_bytes),
312
+ "as_of": _iso_ceil(self.as_of),
313
+ "ledger": list(self.ledger),
314
+ "eligible": bool(self.eligible),
315
+ "next_attempt_at": _iso_ceil(self.next_attempt_at),
316
+ "continuation_cutoff": self.continuation_cutoff,
317
+ "last_failure": self.last_failure,
318
+ "progress": self.progress,
319
+ "op_seq": int(self.op_seq),
320
+ })
321
+ return record
322
+
323
+
324
+ def _operation_start(visit_at: dt.datetime, op_clock: float,
325
+ visit_clock0: "float | None") -> dt.datetime:
326
+ """The operation's start: the visit instant plus the visit clock's
327
+ elapsed time (never negative), or the visit instant itself."""
328
+ if visit_clock0 is None:
329
+ return visit_at
330
+ return visit_at + dt.timedelta(seconds=max(0.0, op_clock - visit_clock0))
331
+
332
+
333
+ # ── the fixed-size record (spec §5.4 "State write", 901-SR-016) ──────────
334
+ #
335
+ # Every writer stores the record as `json.dumps(record, sort_keys=True)`
336
+ # space-padded to exactly `RECLAIM_RECORD_LENGTH` (L) bytes, so a reclaim
337
+ # chunk's same-key `UPDATE` replaces the cell by one of the same size: SQLite
338
+ # overwrites it in place (`sqlite3BtreeInsert`'s same-size path, audited at
339
+ # 3.37.2 and 3.53.4), dirtying exactly the one leaf that holds the row, with
340
+ # no rebalance and no page allocated or freed. No writer deletes the row.
341
+ #
342
+ # L is chosen from the record's BOUNDED fields (`_bounded_record` clamps and
343
+ # drops everything else): `max_record_length()` serializes the worst case -
344
+ # every integer at its widest, every instant at 20 characters, a 48-character
345
+ # failure reason and 25 ledger buckets - which is 3,101 bytes. L = 3,968 is
346
+ # 1.279 x that (at least 25% headroom), and the whole cell stays local at
347
+ # 4 KiB pages: a 4-byte record header + the 38-byte key + L = 4,010 bytes of
348
+ # payload <= U - 35 = 4,061 (no overflow page). Both are asserted at import.
349
+ RECLAIM_RECORD_LENGTH = 3968
350
+ _RECORD_HEADROOM_NUMERATOR, _RECORD_HEADROOM_DENOMINATOR = 5, 4
351
+ _INT64_MIN, _INT64_MAX = -(2 ** 63), 2 ** 63 - 1
352
+ _POLICY_VERSION_MAX = 999
353
+ _REASON_MAX_CHARS = 48
354
+ _LEDGER_KEEP = 25
355
+ _NONNEGATIVE_INT_KEYS = ("freelist_count", "unreclaimed_bytes", "op_seq")
356
+ _INSTANT_KEYS = ("attempted_at", "as_of", "next_attempt_at",
357
+ "continuation_cutoff")
358
+ #: The pacing instants, rounded UP to a whole second (901-RW-001); the
359
+ #: others are observations and keep the truncated form.
360
+ _CEIL_INSTANT_KEYS = ("as_of", "next_attempt_at")
361
+ _BOOL_KEYS = ("made_progress", "deadline_hit", "eligible")
362
+
363
+
364
+ class StateRecordOversize(ValueError):
365
+ """A serialized record longer than `RECLAIM_RECORD_LENGTH`."""
366
+
367
+
368
+ def _clamp_int(value, low=_INT64_MIN, high=_INT64_MAX) -> "int | None":
369
+ if isinstance(value, bool) or not isinstance(value, int):
370
+ return None
371
+ return max(low, min(high, value))
372
+
373
+
374
+ def _bounded_instant(value, *, ceil: bool = False) -> "str | None":
375
+ parsed = _parse_utc(value)
376
+ if parsed is not None and ceil:
377
+ try:
378
+ parsed = _ceil_second(parsed)
379
+ except OverflowError:
380
+ return None
381
+ if parsed is None or not 1000 <= parsed.year <= 9999:
382
+ return None
383
+ return _budget.iso_z(parsed)
384
+
385
+
386
+ def _bounded_reason(value) -> str:
387
+ if (isinstance(value, str) and 0 < len(value) <= _REASON_MAX_CHARS
388
+ and all(c.islower() or c.isdigit() or c == "_" for c in value)
389
+ and value.isascii()):
390
+ return value
391
+ return "unknown"
392
+
393
+
394
+ def _bounded_record(record: dict) -> dict:
395
+ """The record restricted to its known, bounded fields: what every writer
396
+ stores, so the serialized length can never exceed the computed maximum."""
397
+ out = {}
398
+ for key in _NONNEGATIVE_INT_KEYS:
399
+ if key in record:
400
+ out[key] = _clamp_int(record[key], 0)
401
+ if "balance_bytes" in record:
402
+ out["balance_bytes"] = _clamp_int(record["balance_bytes"])
403
+ if "policy_version" in record:
404
+ out["policy_version"] = _clamp_int(record["policy_version"], 0,
405
+ _POLICY_VERSION_MAX)
406
+ for key in _INSTANT_KEYS:
407
+ if key in record:
408
+ out[key] = _bounded_instant(record[key],
409
+ ceil=key in _CEIL_INSTANT_KEYS)
410
+ for key in _BOOL_KEYS:
411
+ if key in record:
412
+ out[key] = record[key] is True
413
+ if "ledger" in record:
414
+ buckets = _budget.parse_ledger(record["ledger"])[-_LEDGER_KEEP:]
415
+ out["ledger"] = [
416
+ {"hour": _budget.iso_z(b.hour),
417
+ "charged": _clamp_int(b.charged, 0),
418
+ "largest": _clamp_int(b.largest, 0)} for b in buckets]
419
+ if "last_failure" in record:
420
+ failure = record["last_failure"]
421
+ out["last_failure"] = (
422
+ {"reason": _bounded_reason(failure.get("reason")),
423
+ "at": _bounded_instant(failure.get("at"))}
424
+ if isinstance(failure, dict) else None)
425
+ if "progress" in record:
426
+ progress = record["progress"]
427
+ out["progress"] = (
428
+ {"freelist_pages": _clamp_int(progress.get("freelist_pages"), 0),
429
+ "page_size": _clamp_int(progress.get("page_size"), 0),
430
+ "observed_at": _bounded_instant(progress.get("observed_at"))}
431
+ if isinstance(progress, dict) else None)
432
+ return out
433
+
434
+
435
+ def _record_json(record: dict) -> str:
436
+ return json.dumps(_bounded_record(record), sort_keys=True)
437
+
438
+
439
+ def serialize_reclaim_record(record: dict) -> str:
440
+ """The record as exactly `RECLAIM_RECORD_LENGTH` bytes of space-padded
441
+ JSON; raises `StateRecordOversize` rather than store another length."""
442
+ text = _record_json(record)
443
+ if len(text.encode("utf-8")) > RECLAIM_RECORD_LENGTH:
444
+ raise StateRecordOversize(len(text))
445
+ return text + " " * (RECLAIM_RECORD_LENGTH - len(text.encode("utf-8")))
446
+
447
+
448
+ def max_record_length() -> int:
449
+ """The longest record `_bounded_record` can produce."""
450
+ instant = "9999-12-31T23:59:59Z"
451
+ widest_nonnegative = _INT64_MAX
452
+ worst = {key: widest_nonnegative for key in _NONNEGATIVE_INT_KEYS}
453
+ worst.update({key: instant for key in _INSTANT_KEYS})
454
+ worst.update({key: False for key in _BOOL_KEYS})
455
+ worst.update({
456
+ "balance_bytes": _INT64_MIN,
457
+ "policy_version": _POLICY_VERSION_MAX,
458
+ "ledger": [{"hour": f"9999-12-{1 + h // 24:02d}T{h % 24:02d}:00:00Z",
459
+ "charged": widest_nonnegative,
460
+ "largest": widest_nonnegative}
461
+ for h in range(_LEDGER_KEEP)],
462
+ "last_failure": {"reason": "x" * _REASON_MAX_CHARS, "at": instant},
463
+ "progress": {"freelist_pages": widest_nonnegative,
464
+ "page_size": widest_nonnegative, "observed_at": instant},
465
+ })
466
+ return len(_record_json(worst).encode("utf-8"))
467
+
468
+
469
+ def _varint_size(value: int) -> int:
470
+ size = 1
471
+ while value >= 0x80 and size < 9:
472
+ value >>= 7
473
+ size += 1
474
+ return size
475
+
476
+
477
+ def record_cell_payload_bytes(length: int = RECLAIM_RECORD_LENGTH) -> int:
478
+ """The `cache_meta(key, value)` record payload of the #780 row: header
479
+ (its size varint + the key's and the value's serial types), key, value."""
480
+ key = len(RECLAIM_PENDING_KEY.encode("utf-8"))
481
+ types = _varint_size(2 * key + 13) + _varint_size(2 * length + 13)
482
+ header = types + 1
483
+ return _varint_size(header) - 1 + header + key + length
484
+
485
+
486
+ def record_fits_locally(usable_size: int,
487
+ length: int = RECLAIM_RECORD_LENGTH) -> bool:
488
+ """A table-leaf cell keeps its whole payload local up to U - 35 bytes."""
489
+ return record_cell_payload_bytes(length) <= int(usable_size) - 35
490
+
491
+
492
+ assert RECLAIM_RECORD_LENGTH * _RECORD_HEADROOM_DENOMINATOR >= (
493
+ max_record_length() * _RECORD_HEADROOM_NUMERATOR), (
494
+ "the #780 record length needs 25% headroom over its largest record")
495
+ assert record_fits_locally(4096), (
496
+ "the #780 record cell must stay local on a 4 KiB page")
497
+
498
+
499
+ def read_reclaim_pending(conn: sqlite3.Connection) -> "dict | None":
500
+ """The durable #780/#901 record (padding stripped), or None when absent
501
+ or unreadable."""
502
+ try:
503
+ row = conn.execute(
504
+ "SELECT value FROM cache_meta WHERE key=?",
505
+ (RECLAIM_PENDING_KEY,),
506
+ ).fetchone()
507
+ except sqlite3.Error:
508
+ return None
509
+ if row is None or not row[0]:
510
+ return None
511
+ try:
512
+ state = json.loads(str(row[0]).rstrip(" "))
513
+ except (TypeError, ValueError):
514
+ return None
515
+ return state if isinstance(state, dict) else None
516
+
517
+
518
+ def reclaim_record_length(conn: sqlite3.Connection) -> "int | None":
519
+ """The stored record's length in bytes, or None when the row is absent."""
520
+ row = conn.execute(
521
+ "SELECT length(CAST(value AS BLOB)) FROM cache_meta WHERE key=?",
522
+ (RECLAIM_PENDING_KEY,),
523
+ ).fetchone()
524
+ return None if row is None or row[0] is None else int(row[0])
525
+
526
+
527
+ def _write_reclaim_pending(conn: sqlite3.Connection, state: "dict | None") -> None:
528
+ """Store the record at exactly L bytes. Nothing deletes the row: `None`
529
+ (an ended episode) rewrites the current record with `eligible` false."""
530
+ if state is None:
531
+ current = read_reclaim_pending(conn) or {}
532
+ state = {**current, "eligible": False}
533
+ conn.execute(
534
+ "INSERT INTO cache_meta(key, value) VALUES (?, ?) "
535
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
536
+ (RECLAIM_PENDING_KEY, serialize_reclaim_record(state)),
219
537
  )
220
538
 
221
539
 
540
+ def _write_reclaim_state_in_chunk(conn: sqlite3.Connection,
541
+ record: dict) -> None:
542
+ """A reclaim chunk's state write: a same-key `UPDATE` of the existing
543
+ L-byte row, never an insert or `REPLACE`, so it overwrites one cell in
544
+ place. Raises `StateRecordOversize` before writing anything."""
545
+ value = serialize_reclaim_record(record)
546
+ cursor = conn.execute("UPDATE cache_meta SET value=? WHERE key=?",
547
+ (value, RECLAIM_PENDING_KEY))
548
+ if cursor.rowcount != 1:
549
+ raise sqlite3.OperationalError("the #780 record row is missing")
550
+
551
+
552
+ def normalize_reclaim_record(conn: sqlite3.Connection,
553
+ now_utc: dt.datetime) -> bool:
554
+ """Write the record at length L when it is absent (an older binary's
555
+ deleted-row state) or of another length (a legacy or pre-#901 record),
556
+ in its own state-only transaction outside any chunk. Uncharged, and
557
+ measured under I2 like every other maintenance state write. Returns True
558
+ when it wrote. `conn` must hold no open transaction."""
559
+ conn.execute("BEGIN IMMEDIATE")
560
+ try:
561
+ if reclaim_record_length(conn) == RECLAIM_RECORD_LENGTH:
562
+ conn.rollback()
563
+ return False
564
+ record = read_reclaim_pending(conn)
565
+ if not isinstance(record, dict) or record.get("policy_version") is None:
566
+ record = PacingState.from_record(record, now_utc).to_record()
567
+ _write_reclaim_pending(conn, record)
568
+ conn.commit()
569
+ return True
570
+ except BaseException:
571
+ conn.rollback()
572
+ raise
573
+
574
+
575
+ def _ceil_div(numerator: int, denominator: int) -> int:
576
+ return -(-int(numerator) // int(denominator))
577
+
578
+
579
+ def page_frame_bytes(page_size: int) -> int:
580
+ """One distinct dirty page's whole cost: its WAL frame (page + 24-byte
581
+ header) and its eventual checkpoint copy (spec I4)."""
582
+ return 2 * int(page_size) + 24
583
+
584
+
585
+ def pointer_map_page_bytes(page_size: int) -> int:
586
+ """B: 1.25 x (2P + 24) rounded up to a KiB, from the store's page size."""
587
+ return _ceil_div(POINTER_MAP_FACTOR_NUMERATOR * page_frame_bytes(page_size),
588
+ POINTER_MAP_FACTOR_DENOMINATOR * KiB) * KiB
589
+
590
+
591
+ def pointer_map_cap(page_count: int, usable_size: int, rows: int,
592
+ page_size: int) -> int:
593
+ """I4's M_cap = ceil(N_max / (floor(U/5) + 1)) + 1, where N_max is the
594
+ page count at the operation's start plus ceil((512 KiB + 12 KiB x rows)
595
+ / P): every other dirty page appended at the end of the file (Q8)."""
596
+ growth = _ceil_div(I4_DELETION_FIXED_BYTES
597
+ + I4_DELETION_PER_ROW_BYTES * max(0, int(rows)),
598
+ int(page_size))
599
+ n_max = int(page_count) + growth
600
+ return _ceil_div(n_max, int(usable_size) // 5 + 1) + 1
601
+
602
+
603
+ def deletion_reservation(rows: int, *, page_count: int, usable_size: int,
604
+ page_size: int) -> int:
605
+ """F + A x rows + B x M_cap (spec §5.4, Q8), from geometry read inside
606
+ the operation's `BEGIN IMMEDIATE`."""
607
+ rows = max(0, int(rows))
608
+ per_row = (DELETION_PER_ROW_FALLBACK_BYTES
609
+ if rows > DELETION_FALLBACK_ROW_THRESHOLD
610
+ else DELETION_PER_ROW_BYTES)
611
+ return (DELETION_FIXED_BYTES + per_row * rows
612
+ + pointer_map_page_bytes(page_size)
613
+ * pointer_map_cap(page_count, usable_size, rows, page_size))
614
+
615
+
616
+ def _record_int(op: dict, key: str) -> "int | None":
617
+ value = op.get(key)
618
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
619
+ return None
620
+ return value
621
+
622
+
623
+ def recompute_charge(op: dict) -> "int | None":
624
+ """The reservation an operation record's own inputs imply, or None when
625
+ the record lacks them or carries another formula version. `op` is an
626
+ operation payload (`as_phase_payload`) or a published maintenance record
627
+ with its `phase` ("delete" | "reclaim")."""
628
+ if not isinstance(op, dict):
629
+ return None
630
+ if op.get("reservation_version") != RESERVATION_VERSION:
631
+ return None
632
+ phase = op.get("phase") or op.get("kind")
633
+ if phase == "delete":
634
+ values = [_record_int(op, k) for k in (
635
+ "rows", "page_count", "usable_size", "page_size")]
636
+ if None in values or values[3] <= 0 or values[2] <= 0:
637
+ return None
638
+ rows, page_count, usable_size, page_size = values
639
+ return deletion_reservation(rows, page_count=page_count,
640
+ usable_size=usable_size,
641
+ page_size=page_size)
642
+ if phase == "reclaim":
643
+ values = [_record_int(op, k) for k in (
644
+ "identified_pages", "unidentified_pages", "page_size",
645
+ "fixed_bytes")]
646
+ if None in values or values[2] <= 0 or values[0] < 1:
647
+ return None
648
+ identified, unidentified, page_size, fixed = values
649
+ return reclaim_reservation(identified, unidentified,
650
+ page_size=page_size, fixed_bytes=fixed)
651
+ return None
652
+
653
+
654
+ def reclaim_reservation(identified: int, unidentified: int, *,
655
+ page_size: int,
656
+ fixed_bytes: int = RECLAIM_FIXED_BYTES) -> int:
657
+ """(|ID| + UNID + 1) x (2P + 24) + F_r from a chunk's plan (Q9)."""
658
+ return _planner.reservation_bytes(identified, unidentified, page_size,
659
+ fixed_bytes)
660
+
661
+
662
+ # ── store geometry, eligibility and the ceiling ──────────────────────────
663
+
664
+ @dataclass(frozen=True)
665
+ class StoreGeometry:
666
+ page_count: int
667
+ freelist_count: int
668
+ page_size: int
669
+
670
+ @property
671
+ def free_bytes(self) -> int:
672
+ return self.freelist_count * self.page_size
673
+
674
+
675
+ #: `SQLITE_FCNTL_RESERVE_BYTES` (sqlite3.h): a negative argument only reads
676
+ #: the bytes reserved at the end of every page.
677
+ _SQLITE_FCNTL_RESERVE_BYTES = 38
678
+ #: The most a page can reserve: the conservative usable size when the
679
+ #: reserve cannot be read (a smaller U only raises M_cap and the charge).
680
+ _MAX_RESERVED_BYTES = 255
681
+
682
+
683
+ def _sqlite_handle(conn) -> "int | None":
684
+ """The `sqlite3 *` behind a CPython connection (through test wrappers
685
+ that delegate via `_conn`), as `bin/_lib_sqlite_close.py` reaches it."""
686
+ real = conn
687
+ for _ in range(4):
688
+ if isinstance(real, sqlite3.Connection):
689
+ break
690
+ real = getattr(real, "_conn", None)
691
+ if (not isinstance(real, sqlite3.Connection)
692
+ or sys.implementation.name != "cpython"):
693
+ return None
694
+ import ctypes
695
+
696
+ return ctypes.c_void_p.from_address(
697
+ id(real) + 2 * ctypes.sizeof(ctypes.c_void_p)).value or None
698
+
699
+
700
+ def reserved_bytes(conn) -> "int | None":
701
+ """Bytes reserved per page of `main`, read through the stdlib
702
+ extension's own SQLite (never a raw descriptor of the database file,
703
+ whose close would drop this process's POSIX locks); None when it cannot
704
+ be read."""
705
+ handle = _sqlite_handle(conn)
706
+ if handle is None:
707
+ return None
708
+ try:
709
+ import _sqlite3
710
+ import ctypes
711
+
712
+ file_control = ctypes.CDLL(_sqlite3.__file__).sqlite3_file_control
713
+ file_control.argtypes = (ctypes.c_void_p, ctypes.c_char_p,
714
+ ctypes.c_int, ctypes.c_void_p)
715
+ file_control.restype = ctypes.c_int
716
+ value = ctypes.c_int(-1)
717
+ rc = file_control(ctypes.c_void_p(handle), b"main",
718
+ _SQLITE_FCNTL_RESERVE_BYTES, ctypes.byref(value))
719
+ except (OSError, AttributeError, ImportError):
720
+ return None
721
+ if rc != 0 or not 0 <= value.value <= _MAX_RESERVED_BYTES:
722
+ return None
723
+ return int(value.value)
724
+
725
+
726
+ @dataclass(frozen=True)
727
+ class DeletionGeometry:
728
+ """The Q8 reservation inputs, read inside the operation's transaction."""
729
+ page_count: int
730
+ page_size: int
731
+ usable_size: int
732
+
733
+
734
+ def deletion_geometry(conn: sqlite3.Connection) -> DeletionGeometry:
735
+ page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
736
+ page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
737
+ reserved = reserved_bytes(conn)
738
+ usable = page_size - (_MAX_RESERVED_BYTES if reserved is None else reserved)
739
+ return DeletionGeometry(page_count, page_size, usable)
740
+
741
+
742
+ def store_geometry(conn: sqlite3.Connection) -> "StoreGeometry | None":
743
+ try:
744
+ page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
745
+ freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
746
+ page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
747
+ except (sqlite3.Error, TypeError, ValueError):
748
+ return None
749
+ if page_count <= 0 or page_size <= 0 or freelist < 0:
750
+ return None
751
+ return StoreGeometry(page_count, freelist, page_size)
752
+
753
+
754
+ def reclaim_episode_active(geometry: "StoreGeometry | None",
755
+ episode_active: bool) -> bool:
756
+ """Start above 2 GiB AND 20%; an active episode stops at 1 GiB OR 10%."""
757
+ if geometry is None:
758
+ return False
759
+ free = geometry.free_bytes
760
+ ratio = geometry.freelist_count / geometry.page_count
761
+ if episode_active:
762
+ return free > RECLAIM_STOP_BYTES and ratio > RECLAIM_STOP_RATIO
763
+ return free > RECLAIM_START_BYTES and ratio > RECLAIM_START_RATIO
764
+
765
+
766
+ def observed_reclaim_backlog_bytes(conn: sqlite3.Connection) -> "int | None":
767
+ geometry = store_geometry(conn)
768
+ return None if geometry is None else geometry.free_bytes
769
+
770
+
771
+ def backlog_over_ceiling(backlog_bytes: "int | None") -> bool:
772
+ return backlog_bytes is not None and backlog_bytes >= RECLAIM_CEILING_BYTES
773
+
774
+
775
+ def observed_backlog_over_ceiling(conn: sqlite3.Connection) -> bool:
776
+ """Doctor FAIL and the rebuild refusal: the OBSERVED freelist."""
777
+ return backlog_over_ceiling(observed_reclaim_backlog_bytes(conn))
778
+
779
+
780
+ # ── connections ───────────────────────────────────────────────────────────
781
+
782
+ def _apply_connection_settings(conn: sqlite3.Connection) -> None:
783
+ conn.execute(f"PRAGMA busy_timeout={_CONNECTION_BUSY_TIMEOUT_MS}")
784
+ conn.execute(f"PRAGMA synchronous={_CONNECTION_SYNCHRONOUS}")
785
+ conn.execute(f"PRAGMA journal_size_limit={_CONNECTION_JOURNAL_SIZE_LIMIT}")
786
+
787
+
788
+ def open_deletion_connection(db_path) -> sqlite3.Connection:
789
+ """The per-visit deletion connection: 256 MiB cache, in-memory temp
790
+ store, SQLite's automatic checkpoint and default close. Closed at the end
791
+ of the visit, so the enlarged cache never outlives it."""
792
+ conn = sqlite3.connect(str(db_path),
793
+ timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
794
+ try:
795
+ _apply_connection_settings(conn)
796
+ conn.execute(f"PRAGMA cache_size = -{DELETION_CACHE_SIZE_KIB}")
797
+ conn.execute("PRAGMA temp_store = MEMORY")
798
+ except BaseException:
799
+ conn.close()
800
+ raise
801
+ return conn
802
+
803
+
804
+ class ReclaimUnavailable(Exception):
805
+ """Reclaim cannot run without checkpointing; carries a typed reason."""
806
+
807
+ def __init__(self, reason: str):
808
+ super().__init__(reason)
809
+ self.reason = reason
810
+
811
+
812
+ def open_reclaim_connection(db_path) -> sqlite3.Connection:
813
+ """A connection that never checkpoints: not at a commit
814
+ (`wal_autocheckpoint = 0`) and not at its close, even as the last
815
+ connection (no-checkpoint-on-close). Raises `ReclaimUnavailable` when the
816
+ close setting cannot be applied.
817
+
818
+ Spec §5.4 (revision 8): `cache_spill = OFF` and `temp_store = MEMORY`, so
819
+ no page of a chunk reaches the WAL before its commit and no statement
820
+ journal touches disk."""
821
+ conn = sqlite3.connect(str(db_path),
822
+ timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
823
+ try:
824
+ try:
825
+ _apply_connection_settings(conn)
826
+ conn.execute("PRAGMA cache_spill = OFF")
827
+ conn.execute("PRAGMA temp_store = MEMORY")
828
+ conn.execute("PRAGMA wal_autocheckpoint = 0")
829
+ applied = (
830
+ int(conn.execute("PRAGMA cache_spill").fetchone()[0]),
831
+ int(conn.execute("PRAGMA temp_store").fetchone()[0]))
832
+ except sqlite3.Error as exc:
833
+ # The primary's disposition 5: a pragma that fails is a typed
834
+ # refusal (§5.4 "a failure to apply the reclaim connection's
835
+ # settings"), never a raw sqlite3.Error from the opener.
836
+ raise ReclaimUnavailable("connection_settings_unavailable") from exc
837
+ if applied != (0, 2):
838
+ raise ReclaimUnavailable("connection_settings_unavailable")
839
+ if int(conn.execute("PRAGMA wal_autocheckpoint").fetchone()[0]) != 0:
840
+ raise ReclaimUnavailable("no_checkpoint_on_close_unavailable")
841
+ try:
842
+ _lib_sqlite_close.set_no_checkpoint_on_close(
843
+ conn, True, purpose="paced reclaim")
844
+ except (sqlite3.Error, OSError, AttributeError, TypeError) as exc:
845
+ raise ReclaimUnavailable(
846
+ "no_checkpoint_on_close_unavailable") from exc
847
+ except BaseException:
848
+ conn.close()
849
+ raise
850
+ return conn
851
+
852
+
853
+ def _open_state_connection(db_path) -> sqlite3.Connection:
854
+ """For a state-only record write (episode end, failure): never
855
+ checkpoints at commit; no-checkpoint-on-close where available."""
856
+ conn = sqlite3.connect(str(db_path),
857
+ timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
858
+ _apply_connection_settings(conn)
859
+ conn.execute("PRAGMA wal_autocheckpoint = 0")
860
+ try:
861
+ _lib_sqlite_close.set_no_checkpoint_on_close(
862
+ conn, True, purpose="retention state")
863
+ except (sqlite3.Error, OSError, AttributeError, TypeError):
864
+ pass
865
+ return conn
866
+
867
+
868
+ # Seams the tests replace to observe each connection a visit opens.
869
+ _open_deletion_connection = open_deletion_connection
870
+ _open_reclaim_connection = open_reclaim_connection
871
+
872
+
873
+ # ── expired groups ────────────────────────────────────────────────────────
874
+
222
875
  def _prunable_groups(
223
876
  conn: sqlite3.Connection, table: str, key_col: str, cutoff: str
224
877
  ) -> list[str]:
225
- """Return group keys (``key_col`` values) with a dated row all before the
226
- cutoff. MAX(timestamp_utc) IS NOT NULL excludes all-NULL-timestamp groups
227
- (conservative — never pruned in isolation, F12). MAX(...) < cutoff is
228
- equivalent to NOT EXISTS a row at/after the cutoff (NULLs are ignored by
229
- MAX and never satisfy ``>= cutoff``)."""
878
+ """Group keys with a dated row all before the cutoff. MAX(timestamp_utc)
879
+ IS NOT NULL excludes all-NULL-timestamp groups (F12)."""
230
880
  sql = (
231
881
  f"SELECT {key_col} FROM {table} "
232
882
  f"WHERE {key_col} IS NOT NULL "
@@ -239,78 +889,568 @@ def _prunable_groups(
239
889
  def _prunable_null_identity_paths(
240
890
  conn: sqlite3.Connection, table: str, key_col: str, cutoff: str
241
891
  ) -> list[str]:
242
- """F12: for rows with a NULL identity column, group by source_path so they
243
- are pruned as a bounded unit rather than orphaned unbounded."""
244
- sql = (
245
- f"SELECT source_path FROM {table} "
246
- f"WHERE {key_col} IS NULL "
247
- f"GROUP BY source_path "
248
- f"HAVING MAX(timestamp_utc) IS NOT NULL AND MAX(timestamp_utc) < ?"
249
- )
250
- return [row[0] for row in conn.execute(sql, (cutoff,))]
892
+ """F12: NULL-identity rows are pruned per source_path.
893
+
894
+ #901 Amendment 19 T1 (spec I1): the per-path ``MAX(timestamp_utc)`` is
895
+ folded in Python over the streamed NULL-identity rows, one entry per
896
+ path, where ``GROUP BY source_path`` sorted every such row through a temp
897
+ b-tree. The class is legacy (current ingest never writes it), but nothing
898
+ bounds how much of it a store retains. Same result: a path qualifies when
899
+ it has a dated row and its latest dated row is before ``cutoff``, in
900
+ ascending path order.
901
+ """
902
+ latest: dict = {}
903
+ for source_path, stamp in conn.execute(
904
+ f"SELECT source_path, timestamp_utc FROM {table} "
905
+ f"WHERE {key_col} IS NULL"
906
+ ):
907
+ if stamp is None:
908
+ continue
909
+ held = latest.get(source_path)
910
+ if held is None or _sqlite_order(stamp) > _sqlite_order(held):
911
+ latest[source_path] = stamp
912
+ bound = _sqlite_order(cutoff)
913
+ return [
914
+ path for path in sorted(latest, key=_sqlite_order)
915
+ if _sqlite_order(latest[path]) < bound
916
+ ]
917
+
918
+
919
+ def _sqlite_order(value) -> tuple:
920
+ """SQLite's cross-type collation for ``MAX``/``<``/``ORDER BY`` on one
921
+ value: NULL < numbers < text (BINARY) < blobs."""
922
+ if value is None:
923
+ return (0, 0)
924
+ if isinstance(value, (int, float)):
925
+ return (1, value)
926
+ if isinstance(value, str):
927
+ return (2, value.encode("utf-8"))
928
+ return (3, bytes(value))
929
+
930
+
931
+ @dataclass(frozen=True)
932
+ class ExpiredGroup:
933
+ provider: str # "claude" | "codex"
934
+ by_path: bool # NULL identity, grouped by source_path (F12)
935
+ value: str
936
+
937
+
938
+ _GROUP_TABLES = {
939
+ "claude": ("conversation_messages", "session_id"),
940
+ "codex": ("codex_conversation_events", "conversation_key"),
941
+ }
942
+
943
+
944
+ def expired_groups(conn: sqlite3.Connection, cutoff_iso: str) -> "list[ExpiredGroup]":
945
+ """Claude groups first, then Codex: the order the visit deletes them."""
946
+ groups: "list[ExpiredGroup]" = []
947
+ for provider in ("claude", "codex"):
948
+ table, key = _GROUP_TABLES[provider]
949
+ groups += [ExpiredGroup(provider, False, value)
950
+ for value in _prunable_groups(conn, table, key, cutoff_iso)]
951
+ groups += [ExpiredGroup(provider, True, value)
952
+ for value in _prunable_null_identity_paths(
953
+ conn, table, key, cutoff_iso)]
954
+ return groups
955
+
956
+
957
+ def group_row_count(conn: sqlite3.Connection, group: ExpiredGroup) -> int:
958
+ """Rows the reservation charges: `conversation_messages` for a Claude
959
+ group, `codex_conversation_events` for a Codex one."""
960
+ table, key = _GROUP_TABLES[group.provider]
961
+ if group.by_path:
962
+ sql = f"SELECT COUNT(*) FROM {table} WHERE {key} IS NULL AND source_path = ?"
963
+ else:
964
+ sql = f"SELECT COUNT(*) FROM {table} WHERE {key} = ?"
965
+ return int(conn.execute(sql, (group.value,)).fetchone()[0])
966
+
967
+
968
+ def _rowcount(cursor) -> int:
969
+ return cursor.rowcount if cursor.rowcount and cursor.rowcount > 0 else 0
970
+
971
+
972
+ def _delete_claude_session(conn: sqlite3.Connection, session_id: str) -> int:
973
+ conn.execute("DELETE FROM conversation_file_touches WHERE session_id = ?",
974
+ (session_id,))
975
+ deleted = _rowcount(conn.execute(
976
+ "DELETE FROM conversation_messages WHERE session_id = ?", (session_id,)))
977
+ conn.execute("DELETE FROM conversation_ai_titles WHERE session_id = ?",
978
+ (session_id,))
979
+ conn.execute("DELETE FROM conversation_sessions WHERE session_id = ?",
980
+ (session_id,))
981
+ return deleted
982
+
983
+
984
+ def _delete_claude_null_path(conn: sqlite3.Connection, source_path: str) -> int:
985
+ # NULL-session messages carry no session_id-keyed titles/rollup, and
986
+ # conversation_file_touches requires a NOT NULL session_id, so stray
987
+ # touches are deleted by message_id before the messages themselves.
988
+ ids = [row[0] for row in conn.execute(
989
+ "SELECT id FROM conversation_messages "
990
+ "WHERE session_id IS NULL AND source_path = ?", (source_path,))]
991
+ if ids:
992
+ conn.executemany(
993
+ "DELETE FROM conversation_file_touches WHERE message_id = ?",
994
+ [(i,) for i in ids])
995
+ return _rowcount(conn.execute(
996
+ "DELETE FROM conversation_messages "
997
+ "WHERE session_id IS NULL AND source_path = ?", (source_path,)))
998
+
999
+
1000
+ def _delete_codex_conversation_derived(
1001
+ conn: sqlite3.Connection, conversation_key: str
1002
+ ) -> None:
1003
+ """#294 S6: drop one pruned conversation's normalized derived rows in the
1004
+ same transaction as its physical events (§3.2 or-delete, §3.4 partial
1005
+ delete riding the per-row FTS trigger, never `'delete-all'`)."""
1006
+ conn.execute(
1007
+ "DELETE FROM codex_conversation_file_touches WHERE conversation_key = ?",
1008
+ (conversation_key,))
1009
+ conn.execute(
1010
+ "DELETE FROM codex_conversation_messages WHERE conversation_key = ?",
1011
+ (conversation_key,))
1012
+ conn.execute(
1013
+ "DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
1014
+ (conversation_key,))
1015
+
1016
+
1017
+ def _delete_codex_conversation(conn: sqlite3.Connection, key: str) -> int:
1018
+ _delete_codex_conversation_derived(conn, key)
1019
+ return _rowcount(conn.execute(
1020
+ "DELETE FROM codex_conversation_events WHERE conversation_key = ?",
1021
+ (key,)))
1022
+
1023
+
1024
+ def _delete_codex_null_path(conn: sqlite3.Connection, source_path: str) -> int:
1025
+ # NULL-conversation_key events never gained thread identity, so S6 never
1026
+ # normalized them (§4.1) — there are no derived rows to clean up.
1027
+ return _rowcount(conn.execute(
1028
+ "DELETE FROM codex_conversation_events "
1029
+ "WHERE conversation_key IS NULL AND source_path = ?", (source_path,)))
251
1030
 
252
1031
 
253
- def _prune_claude(
1032
+ def _delete_group_rows(conn: sqlite3.Connection, group: ExpiredGroup) -> int:
1033
+ if group.provider == "claude":
1034
+ if group.by_path:
1035
+ return _delete_claude_null_path(conn, group.value)
1036
+ return _delete_claude_session(conn, group.value)
1037
+ if group.by_path:
1038
+ return _delete_codex_null_path(conn, group.value)
1039
+ return _delete_codex_conversation(conn, group.value)
1040
+
1041
+
1042
+ def prune_conversation_transcripts(
254
1043
  conn: sqlite3.Connection,
255
- cutoff: str,
256
1044
  *,
1045
+ cutoff_utc: dt.datetime,
257
1046
  after_group: "Callable[[], None] | None" = None,
258
- ) -> tuple[int, int]:
259
- sessions = 0
260
- messages = 0
261
- for session_id in _prunable_groups(
262
- conn, "conversation_messages", "session_id", cutoff
263
- ):
264
- conn.execute(
265
- "DELETE FROM conversation_file_touches WHERE session_id = ?",
266
- (session_id,),
267
- )
268
- cur = conn.execute(
269
- "DELETE FROM conversation_messages WHERE session_id = ?",
270
- (session_id,),
271
- )
272
- messages += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
273
- conn.execute(
274
- "DELETE FROM conversation_ai_titles WHERE session_id = ?",
275
- (session_id,),
276
- )
277
- conn.execute(
278
- "DELETE FROM conversation_sessions WHERE session_id = ?",
279
- (session_id,),
280
- )
281
- sessions += 1
282
- if after_group is not None:
283
- after_group()
284
- for source_path in _prunable_null_identity_paths(
285
- conn, "conversation_messages", "session_id", cutoff
286
- ):
287
- # NULL-session messages carry no session_id-keyed titles/rollup and
288
- # conversation_file_touches requires a NOT NULL session_id, so delete
289
- # any stray touches by message_id, then the messages themselves.
290
- ids = [
291
- row[0]
292
- for row in conn.execute(
293
- "SELECT id FROM conversation_messages "
294
- "WHERE session_id IS NULL AND source_path = ?",
295
- (source_path,),
296
- )
297
- ]
298
- if ids:
299
- conn.executemany(
300
- "DELETE FROM conversation_file_touches WHERE message_id = ?",
301
- [(i,) for i in ids],
302
- )
303
- cur = conn.execute(
304
- "DELETE FROM conversation_messages "
305
- "WHERE session_id IS NULL AND source_path = ?",
306
- (source_path,),
307
- )
308
- messages += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
309
- sessions += 1
310
- if after_group is not None:
311
- after_group()
312
- return sessions, messages
1047
+ ) -> PruneStats:
1048
+ """The unpaced kernel: prune every expired group inside the caller's
1049
+ transaction. Kept for direct kernel callers; the orchestrator below runs
1050
+ each group as a paced, reserved operation instead."""
1051
+ cutoff = _cutoff_iso(cutoff_utc)
1052
+ claude_sessions, claude_messages = _prune_claude(
1053
+ conn, cutoff, after_group=after_group)
1054
+ codex_conversations, codex_events = _prune_codex(
1055
+ conn, cutoff, after_group=after_group)
1056
+ return PruneStats(
1057
+ claude_sessions=claude_sessions,
1058
+ claude_messages=claude_messages,
1059
+ codex_conversations=codex_conversations,
1060
+ codex_events=codex_events,
1061
+ )
1062
+
1063
+
1064
+ def _prune_provider(conn, provider, cutoff, after_group):
1065
+ groups = 0
1066
+ rows = 0
1067
+ table, key = _GROUP_TABLES[provider]
1068
+ for by_path, values in (
1069
+ (False, _prunable_groups(conn, table, key, cutoff)),
1070
+ (True, _prunable_null_identity_paths(conn, table, key, cutoff))):
1071
+ for value in values:
1072
+ rows += _delete_group_rows(conn, ExpiredGroup(provider, by_path, value))
1073
+ groups += 1
1074
+ if after_group is not None:
1075
+ after_group()
1076
+ return groups, rows
1077
+
1078
+
1079
+ def _prune_claude(conn, cutoff, *, after_group=None) -> "tuple[int, int]":
1080
+ return _prune_provider(conn, "claude", cutoff, after_group)
1081
+
1082
+
1083
+ def _prune_codex(conn, cutoff, *, after_group=None) -> "tuple[int, int]":
1084
+ return _prune_provider(conn, "codex", cutoff, after_group)
1085
+
1086
+
1087
+ # ── operations (spec §5.4 protocol) ───────────────────────────────────────
1088
+
1089
+ @dataclass(frozen=True)
1090
+ class OperationResult:
1091
+ op_id: int
1092
+ kind: str # "delete" | "reclaim"
1093
+ outcome: str # "ok" | "gated" | "no_progress" | "skipped"
1094
+ started_at: dt.datetime
1095
+ duration_s: float
1096
+ rows: int = 0
1097
+ pages: int = 0
1098
+ charged_bytes: int = 0
1099
+ balance_before_bytes: "int | None" = None
1100
+ balance_after_bytes: "int | None" = None
1101
+ mode: str = ""
1102
+ process_write_bytes: "int | None" = None
1103
+ write_status: str = "not_sampled"
1104
+ write_overlap: bool = False
1105
+ skip_reason: "str | None" = None
1106
+ # Q8: the deletion reservation's inputs, so the charge is recomputable
1107
+ # from the record alone (`recompute_charge`).
1108
+ page_count: "int | None" = None
1109
+ usable_size: "int | None" = None
1110
+ page_size: "int | None" = None
1111
+ pointer_map_cap: "int | None" = None
1112
+ reservation_version: "int | None" = None
1113
+ # Q9: a reclaim chunk's plan inputs and outcome (§5.5). The planned page
1114
+ # set itself is receipt-only detail and stays out of the record.
1115
+ planner_version: "int | None" = None
1116
+ sqlite_source_id: "str | None" = None
1117
+ freelist_count: "int | None" = None
1118
+ steps: "int | None" = None
1119
+ kind_free: "int | None" = None
1120
+ kind_overflow: "int | None" = None
1121
+ kind_leaf: "int | None" = None
1122
+ kind_interior: "int | None" = None
1123
+ identified_pages: "int | None" = None
1124
+ unidentified_pages: "int | None" = None
1125
+ fixed_bytes: "int | None" = None
1126
+ plan_digest: "str | None" = None
1127
+ reader_status: "str | None" = None
1128
+ inspected_bytes: "int | None" = None
1129
+ inspection_ms: "int | None" = None
1130
+ freelist_reduction: "int | None" = None
1131
+ page_count_reduction: "int | None" = None
1132
+
1133
+ def as_phase_payload(self) -> dict:
1134
+ return {
1135
+ "op_id": self.op_id,
1136
+ "started_at": _iso(self.started_at),
1137
+ "duration_s": self.duration_s,
1138
+ "rows": self.rows,
1139
+ "pages_reclaimed": self.pages,
1140
+ "charged_bytes": self.charged_bytes,
1141
+ "process_write_bytes": self.process_write_bytes,
1142
+ "write_status": self.write_status,
1143
+ "write_overlap": self.write_overlap,
1144
+ "balance_before_bytes": self.balance_before_bytes,
1145
+ "balance_after_bytes": self.balance_after_bytes,
1146
+ "outcome": self.outcome,
1147
+ "mode": self.mode,
1148
+ "page_count": self.page_count,
1149
+ "usable_size": self.usable_size,
1150
+ "page_size": self.page_size,
1151
+ "pointer_map_cap": self.pointer_map_cap,
1152
+ "reservation_version": self.reservation_version,
1153
+ "skip_reason": self.skip_reason,
1154
+ "planner_version": self.planner_version,
1155
+ "sqlite_source_id": self.sqlite_source_id,
1156
+ "freelist_count": self.freelist_count,
1157
+ "steps": self.steps,
1158
+ "kind_free": self.kind_free,
1159
+ "kind_overflow": self.kind_overflow,
1160
+ "kind_leaf": self.kind_leaf,
1161
+ "kind_interior": self.kind_interior,
1162
+ "identified_pages": self.identified_pages,
1163
+ "unidentified_pages": self.unidentified_pages,
1164
+ "fixed_bytes": self.fixed_bytes,
1165
+ "plan_digest": self.plan_digest,
1166
+ "reader_status": self.reader_status,
1167
+ "inspected_bytes": self.inspected_bytes,
1168
+ "inspection_ms": self.inspection_ms,
1169
+ "freelist_reduction": self.freelist_reduction,
1170
+ "page_count_reduction": self.page_count_reduction,
1171
+ }
1172
+
1173
+
1174
+ def _result(kind, outcome, state, started_at, t0, clock, observation, **kw):
1175
+ return OperationResult(
1176
+ op_id=state.op_seq, kind=kind, outcome=outcome, started_at=started_at,
1177
+ duration_s=max(clock() - t0, 0.0),
1178
+ process_write_bytes=observation.process_write_bytes,
1179
+ write_status=observation.status, write_overlap=observation.overlap,
1180
+ **kw)
1181
+
1182
+
1183
+ def delete_group(conn: sqlite3.Connection, group: ExpiredGroup, *,
1184
+ now_utc: dt.datetime, cutoff_iso: str,
1185
+ clock: "Callable[[], float]" = time.monotonic,
1186
+ visit_clock0: "float | None" = None) -> OperationResult:
1187
+ """One whole expired group as one paced operation (spec §5.4 steps 1–4).
1188
+
1189
+ ``visit_clock0`` is the visit clock at the visit's start: the charge's
1190
+ ledger bucket is the operation's own start, ``now_utc`` plus the elapsed
1191
+ visit time (PR-13 (a)), while the allowance accrues at ``now_utc``."""
1192
+ started_at = now_utc.astimezone(UTC)
1193
+ rows = group_row_count(conn, group)
1194
+ fallback = rows > DELETION_FALLBACK_ROW_THRESHOLD
1195
+ mode = "fallback" if fallback else "spill_free"
1196
+ # temp_store may only change outside a transaction; the provider flocks
1197
+ # the visit holds keep `rows` exact until the DELETE.
1198
+ conn.execute(f"PRAGMA temp_store = {'FILE' if fallback else 'MEMORY'}")
1199
+ interval = _lib_write_io.begin_interval("maintenance")
1200
+ t0 = clock()
1201
+ op_started = _operation_start(started_at, t0, visit_clock0)
1202
+ try:
1203
+ conn.execute("BEGIN IMMEDIATE")
1204
+ state = PacingState.from_record(read_reclaim_pending(conn), started_at)
1205
+ before = state.available(started_at)
1206
+ if before < 0:
1207
+ conn.rollback()
1208
+ observation = _lib_write_io.end_interval(interval)
1209
+ return _result("delete", "gated", state, started_at, t0, clock,
1210
+ observation, balance_before_bytes=before,
1211
+ balance_after_bytes=before, mode=mode)
1212
+ # Q8: the geometry is read inside this transaction, before the
1213
+ # reservation is recorded, so M_cap bounds this operation's file.
1214
+ geometry = deletion_geometry(conn)
1215
+ cap = pointer_map_cap(geometry.page_count, geometry.usable_size,
1216
+ rows, geometry.page_size)
1217
+ reservation = deletion_reservation(
1218
+ rows, page_count=geometry.page_count,
1219
+ usable_size=geometry.usable_size, page_size=geometry.page_size)
1220
+ charged = dataclasses.replace(
1221
+ state.charge(reservation, started_at, started_utc=op_started),
1222
+ continuation_cutoff=cutoff_iso)
1223
+ _write_reclaim_pending(conn, charged.to_record())
1224
+ _delete_group_rows(conn, group)
1225
+ conn.commit()
1226
+ except BaseException:
1227
+ conn.rollback()
1228
+ _lib_write_io.end_interval(interval)
1229
+ raise
1230
+ observation = _lib_write_io.end_interval(interval)
1231
+ _lib_write_io.mark_deletion(observation, rows=rows)
1232
+ return _result("delete", "ok", charged, started_at, t0, clock, observation,
1233
+ rows=rows, charged_bytes=reservation,
1234
+ balance_before_bytes=before,
1235
+ balance_after_bytes=charged.balance_bytes, mode=mode,
1236
+ page_count=geometry.page_count,
1237
+ usable_size=geometry.usable_size,
1238
+ page_size=geometry.page_size, pointer_map_cap=cap,
1239
+ reservation_version=RESERVATION_VERSION)
1240
+
1241
+
1242
+ def _run_incremental_vacuum_chunk(conn: sqlite3.Connection, steps: int) -> None:
1243
+ """Run ``steps`` ``incremental_vacuum(1)`` statements INSIDE the caller's
1244
+ transaction, with no other SQL between them.
1245
+
1246
+ One step per statement: a statement limited to one page is complete
1247
+ after its first step on every Python, so CPython 3.11's single-step
1248
+ zero-column cursor cannot shorten it, and no `executescript` is needed -
1249
+ that call would COMMIT the open transaction first and split the
1250
+ reservation from its mutation."""
1251
+ for _ in range(int(steps)):
1252
+ conn.execute("PRAGMA incremental_vacuum(1)").fetchall()
1253
+
1254
+
1255
+ def _sqlite_version_info() -> tuple:
1256
+ """The runtime SQLite's version (a seam G3q injects)."""
1257
+ return tuple(sqlite3.sqlite_version_info)
1258
+
1259
+
1260
+ class _ChunkRefused(Exception):
1261
+ def __init__(self, reason: str, *, inspected_bytes=None,
1262
+ inspection_ms=None):
1263
+ super().__init__(reason)
1264
+ self.reason = reason
1265
+ self.inspected_bytes = inspected_bytes
1266
+ self.inspection_ms = inspection_ms
1267
+
1268
+
1269
+ def _request_plan(db_path, *, page_size: int, page_count: int,
1270
+ freelist_count: int, max_steps: int,
1271
+ deadline_s: float) -> "_planner.ReclaimPlan":
1272
+ """Plan the chunk in a fresh helper process while this process holds
1273
+ the chunk's `BEGIN IMMEDIATE` (spec §5.4 "Read path"). Raises
1274
+ `_ChunkRefused` with a typed reason; never opens the store itself."""
1275
+ deadline_s = max(_PLANNER_MIN_DEADLINE_SECONDS, float(deadline_s))
1276
+ request = {"db": str(db_path), "pageSize": int(page_size),
1277
+ "pageCount": int(page_count),
1278
+ "freelistCount": int(freelist_count),
1279
+ "maxSteps": int(max_steps),
1280
+ "budgetBytes": RECLAIM_CHUNK_BUDGET_BYTES,
1281
+ "fixedBytes": RECLAIM_FIXED_BYTES,
1282
+ "readBudgetBytes": _PLANNER_READ_BUDGET_BYTES,
1283
+ "deadlineMs": int(deadline_s * 1000)}
1284
+ try:
1285
+ child = subprocess.run(
1286
+ list(_PLANNER_COMMAND) + ["--plan"], input=json.dumps(request),
1287
+ capture_output=True, text=True, close_fds=True,
1288
+ timeout=deadline_s + _PLANNER_STARTUP_SLACK_SECONDS)
1289
+ except subprocess.TimeoutExpired as exc:
1290
+ raise _ChunkRefused("helper_timeout") from exc
1291
+ except OSError as exc:
1292
+ raise _ChunkRefused("helper_failed") from exc
1293
+ if child.returncode != 0:
1294
+ raise _ChunkRefused("helper_failed")
1295
+ try:
1296
+ answer = json.loads(child.stdout)
1297
+ except ValueError as exc:
1298
+ raise _ChunkRefused("helper_failed") from exc
1299
+ if not isinstance(answer, dict):
1300
+ raise _ChunkRefused("helper_failed")
1301
+ if answer.get("ok") is not True:
1302
+ reason = answer.get("reason")
1303
+ raise _ChunkRefused(
1304
+ reason if reason in _planner.REFUSAL_REASONS else "helper_failed",
1305
+ inspected_bytes=_record_int(answer, "inspectedBytes"),
1306
+ inspection_ms=_record_int(answer, "inspectionMs"))
1307
+ try:
1308
+ plan = _planner.ReclaimPlan.from_wire(answer.get("plan"))
1309
+ except (ValueError, TypeError, KeyError) as exc:
1310
+ raise _ChunkRefused("helper_failed") from exc
1311
+ if ((plan.page_size, plan.page_count, plan.freelist_count)
1312
+ != (int(page_size), int(page_count), int(freelist_count))
1313
+ or plan.steps > int(max_steps)
1314
+ or plan.fixed_bytes != RECLAIM_FIXED_BYTES):
1315
+ raise _ChunkRefused("wal_endpoint_mismatch")
1316
+ return plan
1317
+
1318
+
1319
+ def _plan_fields(plan: "_planner.ReclaimPlan") -> dict:
1320
+ return dict(
1321
+ planner_version=_planner.PLANNER_VERSION,
1322
+ page_size=plan.page_size, usable_size=plan.usable_size,
1323
+ page_count=plan.page_count, freelist_count=plan.freelist_count,
1324
+ steps=plan.steps, kind_free=plan.kinds["free"],
1325
+ kind_overflow=plan.kinds["overflow"], kind_leaf=plan.kinds["leaf"],
1326
+ kind_interior=plan.kinds["interior"],
1327
+ identified_pages=len(plan.identified),
1328
+ unidentified_pages=plan.unidentified, fixed_bytes=plan.fixed_bytes,
1329
+ plan_digest=plan.digest, reader_status=plan.reader_status,
1330
+ inspected_bytes=plan.inspected_bytes,
1331
+ inspection_ms=plan.inspection_ms,
1332
+ reservation_version=RESERVATION_VERSION)
1333
+
1334
+
1335
+ def reclaim_chunk(conn: sqlite3.Connection, *, now_utc: dt.datetime,
1336
+ pages: int = RECLAIM_MAX_CHUNK_PAGES,
1337
+ next_attempt_at: "dt.datetime | None" = None,
1338
+ clock: "Callable[[], float]" = time.monotonic,
1339
+ deadline_s: "float | None" = None,
1340
+ visit_clock0: "float | None" = None) -> OperationResult:
1341
+ """One planned reclaim chunk of at most ``pages`` (<= 16) steps as one
1342
+ paced operation (spec §5.4, Q9), on a connection from
1343
+ `open_reclaim_connection` (it never checkpoints).
1344
+
1345
+ In ONE `BEGIN IMMEDIATE` transaction: the record precondition, then the
1346
+ plan (a helper subprocess reads the snapshot this transaction holds),
1347
+ then the planned `incremental_vacuum(1)` steps with no other SQL between
1348
+ the plan's reads and the first step, then the frozen reservation's
1349
+ state write (a same-key `UPDATE`), then `COMMIT`. Any refusal rolls back
1350
+ with no vacuum and no charge and returns `outcome="skipped"` with its
1351
+ typed `skip_reason`."""
1352
+ started_at = now_utc.astimezone(UTC)
1353
+ max_steps = max(1, min(int(pages), RECLAIM_MAX_CHUNK_PAGES))
1354
+ deadline = (_PLANNER_DEADLINE_SECONDS if deadline_s is None
1355
+ else float(deadline_s))
1356
+ interval = _lib_write_io.begin_interval("maintenance")
1357
+ t0 = clock()
1358
+ op_started = _operation_start(started_at, t0, visit_clock0)
1359
+ source_id = None
1360
+ state = PacingState(balance_bytes=0, as_of=started_at)
1361
+ before = None
1362
+
1363
+ def skipped(reason, **fields):
1364
+ conn.rollback()
1365
+ observation = _lib_write_io.end_interval(interval)
1366
+ return _result("reclaim", "skipped", state, started_at, t0, clock,
1367
+ observation, balance_before_bytes=before,
1368
+ balance_after_bytes=before, mode="reclaim",
1369
+ skip_reason=reason, reader_status=fields.pop(
1370
+ "reader_status", reason),
1371
+ sqlite_source_id=source_id, **fields)
313
1372
 
1373
+ try:
1374
+ conn.execute("BEGIN IMMEDIATE")
1375
+ if reclaim_record_length(conn) != RECLAIM_RECORD_LENGTH:
1376
+ # The in-place state write needs the L-byte row; the attempt
1377
+ # normalizes it in its own transaction before any chunk.
1378
+ return skipped("state_record_unnormalized", reader_status=None)
1379
+ state = PacingState.from_record(read_reclaim_pending(conn), started_at)
1380
+ before = state.available(started_at)
1381
+ geometry = store_geometry(conn)
1382
+ free = 0 if geometry is None else geometry.freelist_count
1383
+ if before < 0 or free <= 0:
1384
+ conn.rollback()
1385
+ observation = _lib_write_io.end_interval(interval)
1386
+ return _result("reclaim", "gated" if before < 0 else "no_progress",
1387
+ state, started_at, t0, clock, observation,
1388
+ balance_before_bytes=before,
1389
+ balance_after_bytes=before, mode="reclaim")
1390
+ source_id = str(conn.execute(
1391
+ "SELECT sqlite_source_id()").fetchone()[0])[:96]
1392
+ if not _planner.sqlite_audited(_sqlite_version_info()):
1393
+ return skipped("sqlite_unaudited")
1394
+ if int(conn.execute("PRAGMA auto_vacuum").fetchone()[0]) != 2:
1395
+ return skipped("unsupported_geometry")
1396
+ reserved = reserved_bytes(conn)
1397
+ usable = geometry.page_size - (
1398
+ _MAX_RESERVED_BYTES if reserved is None else reserved)
1399
+ if reserved != 0 or not record_fits_locally(usable):
1400
+ return skipped("unsupported_geometry")
1401
+ db_path = _resolve_main_db_path(conn)
1402
+ try:
1403
+ plan = _request_plan(
1404
+ db_path, page_size=geometry.page_size,
1405
+ page_count=geometry.page_count, freelist_count=free,
1406
+ max_steps=max_steps, deadline_s=deadline)
1407
+ except _ChunkRefused as refused:
1408
+ return skipped(refused.reason,
1409
+ inspected_bytes=refused.inspected_bytes,
1410
+ inspection_ms=refused.inspection_ms)
1411
+ # No other SQL between the plan's reads and the first step.
1412
+ _run_incremental_vacuum_chunk(conn, plan.steps)
1413
+ after = store_geometry(conn)
1414
+ reservation = plan.reservation_bytes
1415
+ remaining = after.freelist_count
1416
+ charged = dataclasses.replace(
1417
+ state.charge(reservation, started_at, started_utc=op_started),
1418
+ eligible=True,
1419
+ next_attempt_at=next_attempt_at or state.next_attempt_at,
1420
+ last_failure=None,
1421
+ progress={"freelist_pages": remaining,
1422
+ "page_size": after.page_size,
1423
+ "observed_at": _iso(started_at)},
1424
+ legacy={**state.legacy,
1425
+ "freelist_count": remaining,
1426
+ "unreclaimed_bytes": remaining * after.page_size,
1427
+ "attempted_at": _iso(started_at),
1428
+ "made_progress": True, "deadline_hit": False},
1429
+ )
1430
+ try:
1431
+ _write_reclaim_state_in_chunk(conn, charged.to_record())
1432
+ except StateRecordOversize:
1433
+ return skipped("state_record_oversize", **{
1434
+ k: v for k, v in _plan_fields(plan).items()
1435
+ if k not in ("reader_status",)})
1436
+ conn.commit()
1437
+ except BaseException:
1438
+ conn.rollback()
1439
+ _lib_write_io.end_interval(interval)
1440
+ raise
1441
+ observation = _lib_write_io.end_interval(interval)
1442
+ freelist_reduction = free - remaining
1443
+ return _result("reclaim", "ok", charged, started_at, t0, clock,
1444
+ observation, pages=freelist_reduction,
1445
+ charged_bytes=reservation, balance_before_bytes=before,
1446
+ balance_after_bytes=charged.balance_bytes, mode="reclaim",
1447
+ sqlite_source_id=source_id,
1448
+ freelist_reduction=freelist_reduction,
1449
+ page_count_reduction=geometry.page_count - after.page_count,
1450
+ **_plan_fields(plan))
1451
+
1452
+
1453
+ # ── the visit ─────────────────────────────────────────────────────────────
314
1454
 
315
1455
  def _retention_due(conn: sqlite3.Connection, now_utc: dt.datetime) -> bool:
316
1456
  """True iff the prune has never run or last ran more than 24h ago."""
@@ -340,105 +1480,9 @@ def _stamp_retention_prune(conn: sqlite3.Connection, now_utc: dt.datetime) -> No
340
1480
  )
341
1481
 
342
1482
 
343
- def _run_incremental_vacuum_chunk(conn: sqlite3.Connection, pages: int) -> int:
344
- """Reclaim at most ``pages`` freelist pages; return the freelist count after.
345
-
346
- The one seam every chunk goes through, so a test can replace it to hold the
347
- real SQLite write lock at a barrier and observe what a concurrent reader
348
- does. ``executescript()`` routes through ``sqlite3_exec()``, which steps
349
- zero-column pragma rows through ``SQLITE_DONE`` on Python/SQLite
350
- combinations where ``Cursor.fetchall()`` can stop after the first row
351
- (public Linux 3.11).
352
- """
353
- conn.executescript(f"PRAGMA incremental_vacuum({int(pages)});")
354
- return int(conn.execute("PRAGMA freelist_count").fetchone()[0])
355
-
356
-
357
- def _reclaim_incremental_vacuum(
358
- conn: sqlite3.Connection,
359
- *,
360
- deadline_seconds: float = RECLAIM_DEADLINE_SECONDS,
361
- clock: "Callable[[], float]" = time.monotonic,
362
- ) -> ReclaimOutcome:
363
- """Reclaim freelist pages under a monotonic wall-clock budget (#780).
364
-
365
- The pass runs under the maintenance flock and both provider flocks, so its
366
- cost is a cost every reader and both ingesters pay. It therefore stops at
367
- the deadline and leaves the remainder for the next pass rather than driving
368
- the freelist to zero however long that takes.
369
-
370
- Chunks start small and adapt to measured throughput, so a slow disk cannot
371
- overshoot the budget on the first chunk and a fast one is not held to 64
372
- pages a chunk for the whole pass. The freelist is re-read after every
373
- chunk, which is also how a chunk that reclaimed nothing is detected.
374
- """
375
- page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
376
- before = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
377
- started = clock()
378
- if before <= 0:
379
- return ReclaimOutcome(
380
- freelist_before=before, freelist_after=before, page_size=page_size)
381
- remaining = before
382
- pages = RECLAIM_INITIAL_PAGES
383
- # The SLOWEST rate observed in THIS pass, not the most recent one. A cold
384
- # store's per-page cost is about twenty times its warm cost, so sizing the
385
- # next chunk from the chunk just measured is optimistic exactly when it
386
- # matters: measured on a clone of the production store, that estimator ran
387
- # a first pass to 7.9 s against a 2.0 s budget. Taking the minimum makes
388
- # one slow chunk hold the rest of the pass conservative, and the estimate
389
- # resets each pass, so a warm pass still ramps.
390
- slowest_rate = None
391
- deadline_hit = False
392
- while remaining > 0:
393
- elapsed = clock() - started
394
- if elapsed >= deadline_seconds:
395
- deadline_hit = True
396
- break
397
- requested = min(remaining, pages)
398
- chunk_started = clock()
399
- after = _run_incremental_vacuum_chunk(conn, requested)
400
- chunk_elapsed = max(clock() - chunk_started, 0.0)
401
- if after >= remaining:
402
- # The chunk reclaimed nothing. Continuing would spin against a
403
- # store that is not releasing pages.
404
- remaining = after
405
- break
406
- if chunk_elapsed > 0:
407
- # Aim each chunk at a quarter of the remaining budget, so the pass
408
- # takes several measured steps rather than one guess — and never
409
- # more than DOUBLE the chunk just measured, so a cold first chunk
410
- # cannot extrapolate the pass past its deadline.
411
- rate = (remaining - after) / chunk_elapsed
412
- slowest_rate = (
413
- rate if slowest_rate is None else min(slowest_rate, rate))
414
- budget_left = max(deadline_seconds - (clock() - started), 0.0)
415
- pages = int(max(
416
- RECLAIM_INITIAL_PAGES,
417
- min(RECLAIM_MAX_PAGES,
418
- slowest_rate * budget_left / 4,
419
- requested * RECLAIM_CHUNK_GROWTH_FACTOR),
420
- ))
421
- remaining = after
422
- return ReclaimOutcome(
423
- freelist_before=before,
424
- freelist_after=remaining,
425
- page_size=page_size,
426
- pages_reclaimed=max(0, before - remaining),
427
- duration_s=max(clock() - started, 0.0),
428
- deadline_hit=deadline_hit,
429
- )
430
-
431
-
432
1483
  def _resolve_main_db_path(conn: sqlite3.Connection):
433
- """The file behind this connection's ``main`` schema.
434
-
435
- Read off the connection rather than assumed, because the orchestrator is
436
- called with a conversations connection by the dashboard and with a cache
437
- connection by the from-zero-replay callers; measuring the wrong family's
438
- size would report a reclaim against a file the pass never touched. Falls
439
- back to the conversations path when the connection reports no file, which
440
- is the in-memory case.
441
- """
1484
+ """The file behind this connection's ``main`` schema; the conversations
1485
+ path for an in-memory connection."""
442
1486
  try:
443
1487
  for _seq, name, file_name in conn.execute("PRAGMA database_list"):
444
1488
  if name == "main" and file_name:
@@ -448,149 +1492,202 @@ def _resolve_main_db_path(conn: sqlite3.Connection):
448
1492
  return _cctally_core.CONVERSATIONS_DB_PATH
449
1493
 
450
1494
 
451
- def _db_family_bytes(path) -> int:
452
- """Database plus WAL plus shm, in bytes. Missing members count as zero."""
453
- total = 0
454
- base = str(path)
455
- for member in (base, base + "-wal", base + "-shm"):
456
- try:
457
- total += os.path.getsize(member)
458
- except OSError:
459
- continue
460
- return total
1495
+ def _report(record_phase, result: OperationResult) -> None:
1496
+ if record_phase is not None:
1497
+ record_phase(result.kind, result.as_phase_payload())
461
1498
 
462
1499
 
463
- def _checkpoint_truncate(conn: sqlite3.Connection) -> "tuple[int, int, int] | None":
464
- """`wal_checkpoint(TRUNCATE)`, returning its `(busy, log, checkpointed)` row.
465
-
466
- Completion is PHYSICAL, not a freelist reading. `incremental_vacuum` moves
467
- pages off the freelist and drops `page_count`, but the bytes stay in the
468
- WAL until it is truncated, so clearing the pending state at
469
- `freelist_count == 0` would declare success while the space is still
470
- occupied. Returns None when the checkpoint could not run at all.
471
- """
1500
+ def _update_record(db_path, mutate) -> None:
1501
+ """A state-only record write (no mutation, no charge) on a connection
1502
+ that never checkpoints; `mutate(record) -> record | None` (None = no
1503
+ change, no write)."""
1504
+ conn = _open_state_connection(db_path)
472
1505
  try:
473
- row = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
474
- except sqlite3.Error:
475
- return None
476
- if row is None:
477
- return None
478
- return (int(row[0]), int(row[1]), int(row[2]))
479
-
480
-
481
- def read_reclaim_pending(conn: sqlite3.Connection) -> "dict | None":
482
- """The durable reclaim backlog record, or None when nothing is pending."""
1506
+ conn.execute("BEGIN IMMEDIATE")
1507
+ try:
1508
+ record = read_reclaim_pending(conn)
1509
+ updated = mutate(record)
1510
+ if updated is None:
1511
+ conn.rollback()
1512
+ return
1513
+ _write_reclaim_pending(conn, updated)
1514
+ conn.commit()
1515
+ except BaseException:
1516
+ conn.rollback()
1517
+ raise
1518
+ finally:
1519
+ conn.close()
1520
+
1521
+
1522
+ def _end_reclaim_episode(db_path) -> None:
1523
+ def mutate(record):
1524
+ if not isinstance(record, dict) or record.get("eligible") is not True:
1525
+ return None
1526
+ return {**record, "eligible": False}
1527
+ _update_record(db_path, mutate)
1528
+
1529
+
1530
+ def _record_reclaim_failure(db_path, reason: str, now_utc: dt.datetime) -> None:
1531
+ """Durable only when the reason changes, so repeated skips rewrite
1532
+ nothing."""
1533
+ def mutate(record):
1534
+ record = dict(record) if isinstance(record, dict) else {}
1535
+ prior = record.get("last_failure")
1536
+ if isinstance(prior, dict) and prior.get("reason") == reason:
1537
+ return None
1538
+ if record.get("policy_version") is None:
1539
+ record = PacingState.from_record(record, now_utc).to_record()
1540
+ record["last_failure"] = {"reason": reason, "at": _iso(now_utc)}
1541
+ return record
1542
+ _update_record(db_path, mutate)
1543
+
1544
+
1545
+ def _stamp_retention_complete(conn: sqlite3.Connection, now_utc: dt.datetime) -> None:
1546
+ """Both providers finished: the daily stamp, and the continuation
1547
+ cleared, in one small transaction on the deletion connection."""
1548
+ conn.execute("BEGIN IMMEDIATE")
483
1549
  try:
484
- row = conn.execute(
485
- "SELECT value FROM cache_meta WHERE key=?",
486
- (RECLAIM_PENDING_KEY,),
487
- ).fetchone()
488
- except sqlite3.Error:
489
- return None
490
- if row is None or not row[0]:
491
- return None
1550
+ _stamp_retention_prune(conn, now_utc)
1551
+ record = read_reclaim_pending(conn)
1552
+ if isinstance(record, dict) and record.get("continuation_cutoff") is not None:
1553
+ _write_reclaim_pending(conn, {**record, "continuation_cutoff": None})
1554
+ conn.commit()
1555
+ except BaseException:
1556
+ conn.rollback()
1557
+ raise
1558
+
1559
+
1560
+ def _run_deletion_visit(db_path, *, now, retention_days, state, record_phase,
1561
+ clock) -> PruneStats:
1562
+ visit_clock0 = clock()
1563
+ cutoff_iso = state.continuation_cutoff or _cutoff_iso(
1564
+ now - dt.timedelta(days=int(retention_days)))
1565
+ counts = {"claude_sessions": 0, "claude_messages": 0,
1566
+ "codex_conversations": 0, "codex_events": 0}
1567
+ complete = True
1568
+ conn = _open_deletion_connection(db_path)
492
1569
  try:
493
- state = json.loads(row[0])
494
- except (TypeError, ValueError):
495
- return None
496
- return state if isinstance(state, dict) else None
1570
+ groups = expired_groups(conn, cutoff_iso)
1571
+ for index, group in enumerate(groups):
1572
+ result = delete_group(conn, group, now_utc=now,
1573
+ cutoff_iso=cutoff_iso, clock=clock,
1574
+ visit_clock0=visit_clock0)
1575
+ _report(record_phase, result)
1576
+ if result.outcome != "ok":
1577
+ complete = False
1578
+ break
1579
+ if group.provider == "claude":
1580
+ counts["claude_sessions"] += 1
1581
+ counts["claude_messages"] += result.rows
1582
+ else:
1583
+ counts["codex_conversations"] += 1
1584
+ counts["codex_events"] += result.rows
1585
+ if (result.balance_after_bytes is not None
1586
+ and result.balance_after_bytes < 0
1587
+ and index + 1 < len(groups)):
1588
+ complete = False
1589
+ break
1590
+ if complete:
1591
+ _stamp_retention_complete(conn, now)
1592
+ finally:
1593
+ conn.close()
1594
+ return PruneStats(**counts, complete=complete)
497
1595
 
498
1596
 
499
- def _write_reclaim_pending(conn: sqlite3.Connection, state: "dict | None") -> None:
500
- if state is None:
501
- conn.execute("DELETE FROM cache_meta WHERE key=?", (RECLAIM_PENDING_KEY,))
502
- return
503
- conn.execute(
504
- "INSERT INTO cache_meta(key, value) VALUES (?, ?) "
505
- "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
506
- (RECLAIM_PENDING_KEY, json.dumps(state, sort_keys=True)),
507
- )
1597
+ #: A refused attempt writes nothing durable, so its next attempt time is
1598
+ #: kept here, per store, to hold the one-minute cadence (spec §5.4). A durable
1599
+ #: failure (the reclaim connection unavailable, a SQLite error) records its
1600
+ #: reason only when it changes and writes no `next_attempt_at`, so it holds
1601
+ #: the same cadence here too (#901 Amendment 19 PR-4).
1602
+ _REFUSED_UNTIL: "dict[str, dt.datetime]" = {}
1603
+
1604
+ #: The reasons that come from the planner and its helper (in-memory only);
1605
+ #: the connection-settings and SQLite-error skips stay durable, as before.
1606
+ _IN_MEMORY_SKIP_REASONS = frozenset(_planner.REFUSAL_REASONS) | {
1607
+ "helper_failed", "helper_timeout", "state_record_unnormalized",
1608
+ "state_record_oversize"}
1609
+
508
1610
 
1611
+ def _refusal_due(db_path, now: dt.datetime) -> bool:
1612
+ until = _REFUSED_UNTIL.get(str(db_path))
1613
+ return until is None or now >= until
509
1614
 
510
- def reclaim_backlog_bytes(state: "dict | None") -> int:
511
- """Unreclaimed bytes a pending record reports, or 0 when nothing is."""
512
- if not state:
513
- return 0
1615
+
1616
+ def _publish_refusal(db_path, reason: "str | None", now: dt.datetime) -> None:
1617
+ """The in-memory pass record of a refused attempt (no durable write):
1618
+ `writeIo.reclaimRefusal` for the doctor, and the next attempt time."""
514
1619
  try:
515
- return max(0, int(state.get("unreclaimed_bytes") or 0))
516
- except (TypeError, ValueError):
517
- return 0
1620
+ if reason is None:
1621
+ _REFUSED_UNTIL.pop(str(db_path), None)
1622
+ _lib_write_io.clear_reclaim_refusal()
1623
+ else:
1624
+ _REFUSED_UNTIL[str(db_path)] = now + dt.timedelta(
1625
+ seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
1626
+ _lib_write_io.set_reclaim_refusal(reason, now)
1627
+ except Exception: # noqa: BLE001 — telemetry must not fail maintenance
1628
+ pass
518
1629
 
519
1630
 
520
- def reclaim_backlog_escalated(state: "dict | None") -> bool:
521
- """Whether the backlog is large enough to stop waiting a whole day."""
522
- return reclaim_backlog_bytes(state) >= RECLAIM_ESCALATION_BYTES
1631
+ def _hold_after_failure(db_path, now: dt.datetime) -> None:
1632
+ """A durable reclaim failure waits one interval before the next attempt
1633
+ (spec §5.4: "attempted at most once per minute"). In memory, like a
1634
+ refusal's hold, so a repeated failure still rewrites nothing; it publishes
1635
+ no refusal, because the failure is already the record's `last_failure`.
1636
+ The next chunk that runs clears it (`_publish_refusal(..., None, ...)`)."""
1637
+ _REFUSED_UNTIL[str(db_path)] = now + dt.timedelta(
1638
+ seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
523
1639
 
524
1640
 
525
- def reclaim_backlog_over_ceiling(state: "dict | None") -> bool:
526
- """Whether the backlog has reached the hard ceiling: doctor FAIL, and a new
527
- rebuild is refused because it would add staging churn to a store that is
528
- already failing to drain."""
529
- return reclaim_backlog_bytes(state) >= RECLAIM_CEILING_BYTES
1641
+ def _run_reclaim_attempt(db_path, *, now, record_phase, clock) -> None:
1642
+ next_attempt = now + dt.timedelta(seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
1643
+ try:
1644
+ conn = _open_reclaim_connection(db_path)
1645
+ except ReclaimUnavailable as exc:
1646
+ _hold_after_failure(db_path, now)
1647
+ _record_reclaim_failure(db_path, exc.reason, now)
1648
+ _report(record_phase, OperationResult(
1649
+ op_id=0, kind="reclaim", outcome="skipped", started_at=now,
1650
+ duration_s=0.0, mode="reclaim", skip_reason=exc.reason,
1651
+ reader_status=exc.reason))
1652
+ return
1653
+ started = clock()
1654
+ try:
1655
+ # §5.4 "State write": a legacy-length or absent record is normalized
1656
+ # in its own uncharged transaction before the first chunk.
1657
+ normalize_reclaim_record(conn, now)
1658
+ while clock() - started < RECLAIM_DEADLINE_SECONDS:
1659
+ if not reclaim_episode_active(store_geometry(conn), True):
1660
+ conn.close()
1661
+ conn = None
1662
+ _end_reclaim_episode(db_path)
1663
+ break
1664
+ remaining = RECLAIM_DEADLINE_SECONDS - (clock() - started)
1665
+ result = reclaim_chunk(conn, now_utc=now, next_attempt_at=next_attempt,
1666
+ clock=clock, deadline_s=remaining,
1667
+ visit_clock0=started)
1668
+ _report(record_phase, result)
1669
+ if result.outcome == "skipped" and (
1670
+ result.skip_reason in _IN_MEMORY_SKIP_REASONS):
1671
+ _publish_refusal(db_path, result.skip_reason, now)
1672
+ break
1673
+ if result.outcome == "ok":
1674
+ _publish_refusal(db_path, None, now)
1675
+ if result.outcome != "ok" or (result.balance_after_bytes or 0) < 0:
1676
+ break
1677
+ except sqlite3.Error:
1678
+ _hold_after_failure(db_path, now)
1679
+ _record_reclaim_failure(db_path, "sqlite_error", now)
1680
+ finally:
1681
+ if conn is not None:
1682
+ conn.close()
530
1683
 
531
1684
 
532
- def run_reclaim_pass(
533
- conn: sqlite3.Connection,
534
- *,
535
- now_utc: dt.datetime,
536
- db_path=None,
537
- deadline_seconds: float = RECLAIM_DEADLINE_SECONDS,
538
- clock: "Callable[[], float]" = time.monotonic,
539
- ) -> "dict":
540
- """One bounded reclaim + checkpoint pass, with durable pending state.
541
-
542
- Returns a phase record: ``{reclaim: {...}, checkpoint: {...}, pending: ...}``.
543
- The pending record is retained until a MEASURED reduction in the database
544
- family's size, or the checkpoint's own result, shows the space was really
545
- returned. A pass that made no progress stays pending and is reported; it is
546
- never declared complete.
547
- """
548
- path = _resolve_main_db_path(conn) if db_path is None else db_path
549
- family_before = _db_family_bytes(path)
550
- outcome = _reclaim_incremental_vacuum(
551
- conn, deadline_seconds=deadline_seconds, clock=clock)
552
- checkpoint = _checkpoint_truncate(conn)
553
- family_after = _db_family_bytes(path)
554
- returned = max(0, family_before - family_after)
555
- complete = (
556
- outcome.freelist_drained
557
- and checkpoint is not None
558
- and checkpoint[0] == 0
559
- and checkpoint[1] == 0
560
- )
561
- state = None
562
- if not complete:
563
- state = {
564
- "freelist_count": outcome.freelist_after,
565
- "unreclaimed_bytes": outcome.unreclaimed_bytes,
566
- "attempted_at": now_utc.astimezone(UTC).isoformat(),
567
- "made_progress": bool(outcome.made_progress),
568
- "deadline_hit": bool(outcome.deadline_hit),
569
- }
570
- # Write and commit ONLY when the record actually changed. A pass over a
571
- # store with no backlog would otherwise issue a no-op DELETE and a commit
572
- # on every cycle, which is both pointless and visible to callers counting
573
- # transaction boundaries.
574
- if state != read_reclaim_pending(conn):
575
- _write_reclaim_pending(conn, state)
576
- conn.commit()
577
- return {
578
- "reclaim": {
579
- "pages_reclaimed": outcome.pages_reclaimed,
580
- "freelist_before": outcome.freelist_before,
581
- "freelist_after": outcome.freelist_after,
582
- "duration_s": outcome.duration_s,
583
- "deadline_hit": outcome.deadline_hit,
584
- "made_progress": outcome.made_progress,
585
- },
586
- "checkpoint": {
587
- "result": checkpoint,
588
- "bytes_returned": returned,
589
- "family_bytes": family_after,
590
- },
591
- "pending": state,
592
- "complete": complete,
593
- }
1685
+ def _publish_record(record) -> None:
1686
+ """Hand the latest ledger to this process's `writeIo.maintenance`."""
1687
+ try:
1688
+ _lib_write_io.set_maintenance_record(record)
1689
+ except Exception: # noqa: BLE001 — telemetry must not fail maintenance
1690
+ pass
594
1691
 
595
1692
 
596
1693
  def _maybe_prune_conversation_retention(
@@ -600,31 +1697,29 @@ def _maybe_prune_conversation_retention(
600
1697
  retention_days: int,
601
1698
  force: bool = False,
602
1699
  record_phase: "Callable[[str, dict], None] | None" = None,
1700
+ clock: "Callable[[], float]" = time.monotonic,
603
1701
  ) -> "PruneStats | None":
604
- """Throttled, flock-serialized transcript retention prune (F7 + F9).
605
-
606
- Returns the :class:`PruneStats` when a prune ran, or ``None`` when it was
607
- skipped (retention disabled, throttled within 24h, or a lock contended).
608
-
609
- Concurrency (F7): a dedicated non-blocking MAINTENANCE flock serializes prune
610
- attempts across processes (a second dashboard skips cleanly). It is claimed
611
- EXCLUSIVE and then downgraded to SHARED for the pass proper, so a long prune
612
- cannot starve the fail-closed panel readers that sample it
613
- ``LOCK_SH | LOCK_NB``; a rival ``LOCK_EX | LOCK_NB`` claim still fails
614
- against the held SHARED, so the serialization is unchanged. Under it, the
615
- two provider flocks are taken in a FIXED order (Claude then Codex),
616
- non-blocking, so a rebuild/reingest mid-flight makes the prune skip this
617
- cycle rather than race between candidate selection and deletion. The prune of
618
- each whole session/conversation runs in its own ``BEGIN IMMEDIATE``
619
- transaction (#315), while every flock remains held for the full pass. The
620
- throttle stamp is committed only after both provider phases succeed. A
621
- failure rolls back the active group but preserves completed groups, writes no
622
- stamp, and therefore retries the remainder next cycle.
623
-
624
- ``conn`` must hold no provider flock and no open transaction (the caller
625
- guarantees this — the dashboard opens a dedicated cache connection; the
626
- from-zero-replay callers invoke it after releasing their sync flock, F9).
627
- ``retention_days <= 0`` disables retention (keep forever).
1702
+ """One paced, flock-serialized retention visit (F7 + F9 + #901).
1703
+
1704
+ Returns the deletion's :class:`PruneStats` when a deletion visit ran
1705
+ (``complete`` False on a continuation), or ``None`` when no deletion ran
1706
+ (retention disabled, nothing due, the allowance in debt, a lock
1707
+ contended, or a reclaim-only visit). The one exception is a forced
1708
+ (explicit) visit that is due but finds the allowance in debt: it returns
1709
+ an empty ``PruneStats(complete=False)`` so the caller reports the
1710
+ allowance rather than mistaking the gate for lock contention.
1711
+
1712
+ Concurrency (F7) is unchanged: the MAINTENANCE flock is claimed
1713
+ EXCLUSIVE non-blocking and downgraded to SHARED once both provider flocks
1714
+ (Claude then Codex, non-blocking) are held, so a rival visit or `db
1715
+ vacuum` still cannot claim it while fail-closed panel readers can. Every
1716
+ flock is held for the whole visit. A visit with nothing due, or with the
1717
+ allowance in debt, takes no provider flock and writes nothing.
1718
+
1719
+ ``force`` bypasses only the 24 h throttle (a from-zero replay). It never
1720
+ bypasses the allowance. ``conn`` must hold no provider flock and no open
1721
+ transaction; the visit reads through it and opens its own deletion and
1722
+ reclaim connections. ``retention_days <= 0`` disables retention.
628
1723
  """
629
1724
  if retention_days is None or retention_days <= 0:
630
1725
  return None
@@ -633,20 +1728,31 @@ def _maybe_prune_conversation_retention(
633
1728
  core.APP_DIR.mkdir(parents=True, exist_ok=True)
634
1729
  except OSError:
635
1730
  return None
1731
+ now = now_utc.astimezone(UTC)
1732
+ db_path = _resolve_main_db_path(conn)
636
1733
  maint_fh = open(core.CONVERSATIONS_LOCK_MAINTENANCE_PATH, "w")
637
1734
  try:
638
1735
  try:
639
1736
  fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
640
1737
  except (BlockingIOError, OSError):
641
- return None # another prune holds it; skip cleanly, do NOT stamp
642
- # The daily throttle governs DELETION. A reclaim backlog above the
643
- # escalation threshold still continues on the next cycle, because a
644
- # doctor warning once a day is a report, not an enforcement mechanism.
645
- reclaim_only = False
646
- if not force and not _retention_due(conn, now_utc):
647
- if not reclaim_backlog_escalated(read_reclaim_pending(conn)):
648
- return None
649
- reclaim_only = True
1738
+ return None # another visit holds it; skip cleanly, do NOT stamp
1739
+ record = read_reclaim_pending(conn)
1740
+ _publish_record(record)
1741
+ state = PacingState.from_record(record, now)
1742
+ episode = reclaim_episode_active(store_geometry(conn), state.eligible)
1743
+ episode_ended = state.eligible and not episode
1744
+ deletion_due = (force or state.continuation_cutoff is not None
1745
+ or _retention_due(conn, now))
1746
+ reclaim_due = (episode and (state.next_attempt_at is None
1747
+ or now >= state.next_attempt_at)
1748
+ and _refusal_due(db_path, now))
1749
+ funded = state.available(now) >= 0
1750
+ # An explicit prune gated by the allowance deletes nothing; say so
1751
+ # (an empty continuation), never the lock-contention None.
1752
+ gated = PruneStats(complete=False) if (
1753
+ force and deletion_due and not funded) else None
1754
+ if not episode_ended and not (funded and (deletion_due or reclaim_due)):
1755
+ return gated
650
1756
  claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
651
1757
  try:
652
1758
  try:
@@ -659,93 +1765,25 @@ def _maybe_prune_conversation_retention(
659
1765
  fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
660
1766
  except (BlockingIOError, OSError):
661
1767
  return None # a Codex sync is mid-flight; retry next cycle
662
- # Downgrade the maintenance flock to SHARED for the pass proper.
663
- # The EXCLUSIVE acquire above is what wins the race; holding it
664
- # exclusive for the whole pass additionally locked out the
665
- # fail-CLOSED panel readers, which take this flock
666
- # `LOCK_SH | LOCK_NB` and blank their column on any contention
667
- # (`read_session_titles_bounded` -> every Claude session title in
668
- # the Recent Sessions card, for the minutes a large prune runs).
669
- # A prune is a writer, not a family replacement: concurrent
670
- # writes are already excluded by the two provider flocks, and
671
- # the replacement paths readers guard against take this flock
672
- # EXCLUSIVE, which a held SHARED still blocks. Rival prunes and
673
- # `db vacuum` claim `LOCK_EX | LOCK_NB`, which also still fails.
674
- # MUST stay after both provider flocks: a flock conversion is
675
- # not atomic, so a rival can slip into the downgrade window —
676
- # the provider flocks it then fails to claim are what make that
677
- # harmless.
1768
+ # Downgrade the maintenance flock to SHARED for the visit
1769
+ # proper, AFTER both provider flocks (a flock conversion is not
1770
+ # atomic; the provider flocks make the window harmless).
678
1771
  try:
679
1772
  fcntl.flock(maint_fh, fcntl.LOCK_SH)
680
1773
  except OSError:
681
1774
  pass # keep the exclusive hold; correctness is unchanged
682
- if reclaim_only:
683
- # Deletion is throttled; the backlog is not. Nothing here
684
- # stamps the throttle, so the ordinary daily deletion still
685
- # happens on its own schedule.
686
- try:
687
- phases = run_reclaim_pass(conn, now_utc=now_utc)
688
- except sqlite3.Error:
689
- phases = None
690
- if phases is not None and record_phase is not None:
691
- record_phase("reclaim", phases["reclaim"])
692
- record_phase("checkpoint", phases["checkpoint"])
693
- return None
694
- cutoff = now_utc - dt.timedelta(days=int(retention_days))
695
- conn.execute("BEGIN IMMEDIATE")
696
- try:
697
- def commit_group() -> None:
698
- conn.commit()
699
- conn.execute("BEGIN IMMEDIATE")
700
-
701
- delete_started = time.monotonic()
702
- stats = prune_conversation_transcripts(
703
- conn,
704
- cutoff_utc=cutoff,
705
- after_group=commit_group,
706
- )
707
- _stamp_retention_prune(conn, now_utc)
708
- conn.commit()
709
- except Exception:
710
- conn.rollback()
711
- raise
712
- if record_phase is not None:
713
- record_phase("delete", {
714
- "duration_s": max(
715
- time.monotonic() - delete_started, 0.0),
716
- "claude_messages": stats.claude_messages,
717
- "codex_events": stats.codex_events,
718
- })
719
- # Return the freed pages to the OS. On an INCREMENTAL auto-vacuum
720
- # cache.db (the default for freshly-created DBs, #313 P3) this
721
- # shrinks the file on disk instead of leaving a growing freelist,
722
- # so the transcript prune reclaims space automatically without a
723
- # manual `cctally db vacuum`; on a legacy auto_vacuum=NONE cache it
724
- # is a harmless no-op (those still reclaim via `db vacuum` or a
725
- # `cache-sync --rebuild`). Runs OUTSIDE the committed transaction,
726
- # still under the maintenance + provider flocks (no concurrent
727
- # writer), and best-effort — a reclaim error must never fail the
728
- # already-durable prune.
729
- # Deletion is complete and durable at this point, and only
730
- # deletion completion is stamped. Reclaim is a SEPARATE phase
731
- # with its own budget and its own durable pending state, so a
732
- # pass that runs out of budget continues on the next cycle
733
- # instead of holding three flocks until the freelist drains.
734
- #
735
- # A pass with no new deletion still continues an outstanding
736
- # backlog: the previous implementation reclaimed only when the
737
- # prune had deleted something, so a store that fell behind had
738
- # no way to catch up.
739
- pending_before = read_reclaim_pending(conn)
740
- if stats.total_rows > 0 or pending_before is not None:
741
- try:
742
- phases = run_reclaim_pass(conn, now_utc=now_utc)
743
- except sqlite3.Error:
744
- phases = None
745
- if phases is not None and record_phase is not None:
746
- record_phase("reclaim", phases["reclaim"])
747
- record_phase("checkpoint", phases["checkpoint"])
748
- return stats
1775
+ if episode_ended:
1776
+ _end_reclaim_episode(db_path)
1777
+ stats = None
1778
+ if funded and deletion_due:
1779
+ stats = _run_deletion_visit(
1780
+ db_path, now=now, retention_days=retention_days,
1781
+ state=state, record_phase=record_phase, clock=clock)
1782
+ if funded and reclaim_due:
1783
+ _run_reclaim_attempt(db_path, now=now,
1784
+ record_phase=record_phase, clock=clock)
1785
+ _publish_record(read_reclaim_pending(conn))
1786
+ return stats if stats is not None else gated
749
1787
  finally:
750
1788
  try:
751
1789
  fcntl.flock(codex_fh, fcntl.LOCK_UN)
@@ -764,76 +1802,3 @@ def _maybe_prune_conversation_retention(
764
1802
  except OSError:
765
1803
  pass
766
1804
  maint_fh.close()
767
-
768
-
769
- def _delete_codex_conversation_derived(
770
- conn: sqlite3.Connection, conversation_key: str
771
- ) -> None:
772
- """#294 S6: drop one pruned conversation's normalized derived rows in the same
773
- transaction as its physical events.
774
-
775
- #313 prunes at WHOLE-conversation granularity — ``_prune_codex`` deletes every
776
- ``codex_conversation_events`` row for the key, so every event across every file
777
- of the conversation is gone. The S6 rollup ownership rule (§3.2) is therefore
778
- the "or-delete" branch: with zero surviving normalized rows the rollup is
779
- deleted outright, no recompute needed. The normalized-message DELETE is a
780
- PARTIAL delete over the whole corpus (§3.4), so it rides the per-row FTS
781
- AFTER-DELETE trigger — surviving conversations keep their postings — and NEVER
782
- the full-clear ``'delete-all'``; when ``codex_fts_unavailable`` is set the
783
- triggers are absent and these are plain base deletes. File touches carry an
784
- explicit ``conversation_key`` so they scope exactly. Deleting the derived rows
785
- is part of pruning the SAME conversation, so it does not change the reported
786
- ``codex_conversations`` / ``codex_events`` counts (those stay physical).
787
- """
788
- conn.execute(
789
- "DELETE FROM codex_conversation_file_touches WHERE conversation_key = ?",
790
- (conversation_key,),
791
- )
792
- # Rides the codex_conv_fts_ad per-row trigger (partial delete, §3.4).
793
- conn.execute(
794
- "DELETE FROM codex_conversation_messages WHERE conversation_key = ?",
795
- (conversation_key,),
796
- )
797
- conn.execute(
798
- "DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
799
- (conversation_key,),
800
- )
801
-
802
-
803
- def _prune_codex(
804
- conn: sqlite3.Connection,
805
- cutoff: str,
806
- *,
807
- after_group: "Callable[[], None] | None" = None,
808
- ) -> tuple[int, int]:
809
- conversations = 0
810
- events = 0
811
- for conversation_key in _prunable_groups(
812
- conn, "codex_conversation_events", "conversation_key", cutoff
813
- ):
814
- # #294 S6: purge the derived normalized rows for this conversation in the
815
- # SAME transaction as the events (§3.2 or-delete + §3.4 partial delete).
816
- _delete_codex_conversation_derived(conn, conversation_key)
817
- cur = conn.execute(
818
- "DELETE FROM codex_conversation_events WHERE conversation_key = ?",
819
- (conversation_key,),
820
- )
821
- events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
822
- conversations += 1
823
- if after_group is not None:
824
- after_group()
825
- for source_path in _prunable_null_identity_paths(
826
- conn, "codex_conversation_events", "conversation_key", cutoff
827
- ):
828
- # NULL-conversation_key events never gained thread identity, so S6 never
829
- # normalized them (§4.1) — there are no derived rows to clean up here.
830
- cur = conn.execute(
831
- "DELETE FROM codex_conversation_events "
832
- "WHERE conversation_key IS NULL AND source_path = ?",
833
- (source_path,),
834
- )
835
- events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
836
- conversations += 1
837
- if after_group is not None:
838
- after_group()
839
- return conversations, events