cctally 1.113.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +6035 -722
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +400 -2
- package/bin/_cctally_dashboard.py +98 -20
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +285 -213
- package/bin/_cctally_db.py +1177 -23
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +572 -45
- package/bin/_cctally_five_hour.py +84 -33
- package/bin/_cctally_forecast.py +108 -15
- package/bin/_cctally_journal.py +3950 -116
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +227 -37
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +691 -149
- package/bin/_cctally_rederive.py +320 -41
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +83 -17
- package/bin/_cctally_statusline.py +6154 -517
- package/bin/_cctally_store.py +367 -53
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +61 -56
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +297 -49
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +106 -59
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +714 -69
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +231 -86
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +199 -9
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +319 -82
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +31 -1
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +19 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
"""#313 P3: conversation-transcript retention
|
|
1
|
+
"""#313 P3 + #901: conversation-transcript retention — paced expiry deletion
|
|
2
|
+
and paced reclaim.
|
|
2
3
|
|
|
3
4
|
Prunes ONLY the re-derivable transcript rows:
|
|
4
5
|
* Claude: ``conversation_messages`` + their ``conversation_file_touches`` /
|
|
@@ -19,214 +20,863 @@ least one dated row and NO row at/after the cutoff. Rows whose timestamps are
|
|
|
19
20
|
entirely NULL are treated conservatively and never pruned in isolation (F12).
|
|
20
21
|
NULL identity (``session_id`` / ``conversation_key``) falls back to grouping by
|
|
21
22
|
``source_path`` so malformed rows stay bounded rather than orphaned-unbounded
|
|
22
|
-
(F12).
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
23
|
+
(F12). The FTS5 indexes are external-content and maintained by AFTER-DELETE
|
|
24
|
+
triggers, which stay consistent because whole groups are deleted.
|
|
25
|
+
|
|
26
|
+
#901 (spec §5.4; operator decisions Q2, Q4, Q5, Q7) puts every maintenance
|
|
27
|
+
write on ONE durable allowance:
|
|
28
|
+
|
|
29
|
+
* 4 MiB/min with at most one minute (4 MiB) of credit, accrued by wall clock
|
|
30
|
+
from the durable ``as_of`` and capped, so a restart never refills credit and
|
|
31
|
+
never bypasses the cadence. The balance and the 25-bucket hourly charge
|
|
32
|
+
ledger live in the extended #780 record (``RECLAIM_PENDING_KEY``).
|
|
33
|
+
* An OPERATION is one expired group's deletion or one reclaim chunk. It starts
|
|
34
|
+
only from a non-negative balance and is charged EXACTLY its reservation,
|
|
35
|
+
written in the operation's own ``BEGIN IMMEDIATE`` transaction together with
|
|
36
|
+
its mutation: a crash before COMMIT leaves neither, after it both. Overshoot
|
|
37
|
+
becomes debt and nothing runs until it is repaid. There is no reconciliation
|
|
38
|
+
and no deferred-copy record.
|
|
39
|
+
* Reclaim never checkpoints: its connection runs ``wal_autocheckpoint = 0`` with
|
|
40
|
+
SQLite's no-checkpoint-on-close setting on (plus ``cache_spill = OFF`` and an
|
|
41
|
+
in-memory temp store), and when those settings cannot be applied the attempt
|
|
42
|
+
runs nothing and records why. It starts only when free pages exceed 2 GiB
|
|
43
|
+
AND 20% of the file, stops at 1 GiB OR 10%, and runs at most once a minute
|
|
44
|
+
under the 2 s deadline. Each chunk (Q9) is planned inside its own
|
|
45
|
+
``BEGIN IMMEDIATE`` by `_lib_reclaim_planner` in a helper subprocess, runs at
|
|
46
|
+
most 16 ``incremental_vacuum(1)`` steps, and is charged
|
|
47
|
+
(|ID| + UNID + 1) x (2P + 24) + F_r from that plan; a planner refusal rolls
|
|
48
|
+
back with no vacuum and no charge and is reported in memory only.
|
|
49
|
+
* Deletion keeps whole-group atomicity and SQLite's automatic PASSIVE checkpoint
|
|
50
|
+
at COMMIT, on a per-visit connection with a 256 MiB cache and an in-memory
|
|
51
|
+
temp store, so a group is deleted spill-free; above 60,000 rows the cache cap
|
|
52
|
+
stays and the statement journal uses the file temp store (Q5). Its charge
|
|
53
|
+
(Q8) is F + A x rows + B x M_cap, M_cap the pointer-map worst case of the
|
|
54
|
+
file's geometry read inside the transaction. A visit stops between groups
|
|
55
|
+
when the balance is negative and persists a continuation cutoff; the daily
|
|
56
|
+
stamp is written only when both providers finish.
|
|
57
|
+
* The #780 record is one fixed-size cell (`RECLAIM_RECORD_LENGTH`): every
|
|
58
|
+
writer stores exactly L bytes and none deletes the row, so a chunk's state
|
|
59
|
+
write overwrites it in place and dirties one page.
|
|
31
60
|
"""
|
|
32
61
|
from __future__ import annotations
|
|
33
62
|
|
|
63
|
+
import dataclasses
|
|
34
64
|
import datetime as dt
|
|
35
65
|
import fcntl
|
|
36
66
|
import json
|
|
37
67
|
import os
|
|
38
68
|
import sqlite3
|
|
69
|
+
import subprocess
|
|
70
|
+
import sys
|
|
39
71
|
import time
|
|
40
72
|
from collections.abc import Callable
|
|
41
73
|
from dataclasses import dataclass
|
|
42
74
|
|
|
43
75
|
import _cctally_core
|
|
76
|
+
import _lib_reclaim_planner as _planner
|
|
77
|
+
import _lib_sqlite_close
|
|
78
|
+
import _lib_write_budget as _budget
|
|
79
|
+
import _lib_write_io
|
|
44
80
|
|
|
45
81
|
UTC = dt.timezone.utc
|
|
82
|
+
KiB = 1024
|
|
83
|
+
MiB = 1024 * KiB
|
|
84
|
+
GiB = 1024 * MiB
|
|
46
85
|
|
|
47
86
|
# cache_meta throttle key (framework-untracked KV — NO schema migration, F7).
|
|
48
87
|
_RETENTION_LAST_PRUNE_KEY = "conversation_retention_last_prune_at"
|
|
49
88
|
_RETENTION_THROTTLE_SECONDS = 24 * 60 * 60
|
|
50
89
|
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
# --------------------------------------------------------------------------
|
|
54
|
-
|
|
55
|
-
#: Durable reclaim state (framework-untracked KV, same class as the throttle
|
|
56
|
-
#: key above — a value shape, not a schema change).
|
|
90
|
+
#: The extended #780 record: balance, ledger, episode and progress. The key is
|
|
91
|
+
#: unchanged so an older binary still reads its legacy fields.
|
|
57
92
|
RECLAIM_PENDING_KEY = "conversation_retention_reclaim_pending"
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
#:
|
|
69
|
-
#:
|
|
70
|
-
|
|
71
|
-
#:
|
|
72
|
-
|
|
73
|
-
#:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
#:
|
|
77
|
-
#:
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
#:
|
|
81
|
-
#:
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
#:
|
|
86
|
-
#:
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
#:
|
|
92
|
-
#:
|
|
93
|
-
|
|
94
|
-
#:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
93
|
+
RETENTION_POLICY_VERSION = 1
|
|
94
|
+
_LEGACY_RECORD_KEYS = ("freelist_count", "unreclaimed_bytes", "attempted_at",
|
|
95
|
+
"made_progress", "deadline_hit")
|
|
96
|
+
|
|
97
|
+
# ── Reservation constants: CANDIDATES (spec §5.4, Q8 and Q9) ──────────────
|
|
98
|
+
# Fixed by the §6.3 R-cov calibration: each calibrated part must be at least
|
|
99
|
+
# 1.25 x its measured maximum, rounded up to a KiB. A candidate that falls
|
|
100
|
+
# short is raised, never left below coverage. Calibration edits THIS block
|
|
101
|
+
# only. The analytical page terms (B's 2P + 24 per pointer-map page, and a
|
|
102
|
+
# reclaim chunk's planned pages) are computed, never calibrated downwards.
|
|
103
|
+
#: The formula version every operation record carries; a record with
|
|
104
|
+
#: another version is not recomputed (`recompute_charge`).
|
|
105
|
+
RESERVATION_VERSION = 2
|
|
106
|
+
#: F: a deletion's fixed part per operation (the WAL header is inside it).
|
|
107
|
+
DELETION_FIXED_BYTES = 512 * KiB
|
|
108
|
+
#: A: per deleted row, and above `DELETION_FALLBACK_ROW_THRESHOLD` (Q5).
|
|
109
|
+
DELETION_PER_ROW_BYTES = 15 * KiB
|
|
110
|
+
DELETION_PER_ROW_FALLBACK_BYTES = 50 * KiB
|
|
111
|
+
#: B = ceil(5/4 x (2P + 24) / 1 KiB) KiB per pointer-map page of M_cap, from
|
|
112
|
+
#: the store's own page size (11 KiB at 4 KiB pages).
|
|
113
|
+
POINTER_MAP_FACTOR_NUMERATOR = 5
|
|
114
|
+
POINTER_MAP_FACTOR_DENOMINATOR = 4
|
|
115
|
+
#: F_r: a reclaim chunk's fixed part (WAL header and framing; Q9). The page
|
|
116
|
+
#: term (|ID| + UNID + 1) x (2P + 24) comes from the chunk's own plan.
|
|
117
|
+
RECLAIM_FIXED_BYTES = 64 * KiB
|
|
118
|
+
# ──────────────────────────────────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
#: I4's deletion write bounds (spec §3; limits on the writes themselves,
|
|
121
|
+
#: never calibrated). N_max's growth term is built from the first two.
|
|
122
|
+
I4_DELETION_FIXED_BYTES = 512 * KiB
|
|
123
|
+
I4_DELETION_PER_ROW_BYTES = 12 * KiB
|
|
124
|
+
I4_FALLBACK_PER_ROW_BYTES = 40 * KiB
|
|
125
|
+
|
|
126
|
+
#: Q5: a group above this many rows keeps the cache cap but uses the file
|
|
127
|
+
#: temp store for its statement journal, bounded at 40 KiB per deleted row.
|
|
128
|
+
DELETION_FALLBACK_ROW_THRESHOLD = 60_000
|
|
129
|
+
#: `PRAGMA cache_size = -262144`: a 256 MiB cap, used only as pages dirty.
|
|
130
|
+
DELETION_CACHE_SIZE_KIB = 262_144
|
|
131
|
+
|
|
132
|
+
RECLAIM_START_BYTES = 2 * GiB
|
|
133
|
+
RECLAIM_START_RATIO = 0.20
|
|
134
|
+
RECLAIM_STOP_BYTES = 1 * GiB
|
|
135
|
+
RECLAIM_STOP_RATIO = 0.10
|
|
136
|
+
RECLAIM_MAX_CHUNK_PAGES = 16
|
|
137
|
+
RECLAIM_ATTEMPT_INTERVAL_SECONDS = 60
|
|
138
|
+
#: The secondary deadline between chunks; a chunk itself is not preemptible.
|
|
98
139
|
RECLAIM_DEADLINE_SECONDS = 2.0
|
|
99
|
-
|
|
100
|
-
#:
|
|
101
|
-
|
|
102
|
-
#:
|
|
103
|
-
#:
|
|
104
|
-
|
|
105
|
-
#:
|
|
106
|
-
#:
|
|
107
|
-
#:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
#:
|
|
113
|
-
#:
|
|
114
|
-
#:
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
#:
|
|
119
|
-
#:
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
#:
|
|
123
|
-
|
|
124
|
-
#:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
#:
|
|
128
|
-
#:
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
#: 300 days; at the escalated per-cycle cadence it is under an hour. That
|
|
133
|
-
#: ratio is the reason escalation exists.
|
|
134
|
-
RECLAIM_ESCALATION_BYTES = 256 * 1024 * 1024
|
|
135
|
-
RECLAIM_MIN_PAGES_PER_SECOND = 1.0
|
|
136
|
-
RECLAIM_CEILING_BYTES = 16 * 1024 * 1024 * 1024
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
@dataclass(frozen=True)
|
|
140
|
-
class ReclaimOutcome:
|
|
141
|
-
"""What one bounded reclaim pass observed. Never a claim about the file."""
|
|
142
|
-
|
|
143
|
-
freelist_before: int = 0
|
|
144
|
-
freelist_after: int = 0
|
|
145
|
-
page_size: int = 0
|
|
146
|
-
pages_reclaimed: int = 0
|
|
147
|
-
duration_s: float = 0.0
|
|
148
|
-
deadline_hit: bool = False
|
|
149
|
-
|
|
150
|
-
@property
|
|
151
|
-
def unreclaimed_bytes(self) -> int:
|
|
152
|
-
return max(0, self.freelist_after) * max(0, self.page_size)
|
|
153
|
-
|
|
154
|
-
@property
|
|
155
|
-
def made_progress(self) -> bool:
|
|
156
|
-
if self.pages_reclaimed <= 0:
|
|
157
|
-
return False
|
|
158
|
-
if self.duration_s <= 0:
|
|
159
|
-
return True
|
|
160
|
-
return (self.pages_reclaimed / self.duration_s) >= (
|
|
161
|
-
RECLAIM_MIN_PAGES_PER_SECOND)
|
|
162
|
-
|
|
163
|
-
@property
|
|
164
|
-
def freelist_drained(self) -> bool:
|
|
165
|
-
return self.freelist_after <= 0
|
|
140
|
+
#: Doctor FAIL and rebuild refusal, judged on the OBSERVED freelist; reaching
|
|
141
|
+
#: it never accelerates automatic writes.
|
|
142
|
+
RECLAIM_CEILING_BYTES = 16 * GiB
|
|
143
|
+
#: A chunk's whole reservation target (Q9): the plan grows one step at a
|
|
144
|
+
#: time while it stays at most this; a single step over it runs alone.
|
|
145
|
+
RECLAIM_CHUNK_BUDGET_BYTES = 4 * MiB
|
|
146
|
+
#: Every typed reason a reclaim attempt can skip with (spec §5.4 "Refusal"):
|
|
147
|
+
#: the connection's settings, the record's precondition, the runtime SQLite,
|
|
148
|
+
#: the planner's own refusals and its helper, and an unexpected SQLite error.
|
|
149
|
+
RECLAIM_SKIP_REASONS = (
|
|
150
|
+
"no_checkpoint_on_close_unavailable", "connection_settings_unavailable",
|
|
151
|
+
"sqlite_error", "state_record_unnormalized", "state_record_oversize",
|
|
152
|
+
"helper_failed", "helper_timeout") + _planner.REFUSAL_REASONS
|
|
153
|
+
#: The planner helper: a fresh interpreter (never a fork of this process),
|
|
154
|
+
#: so no raw descriptor of the store is ever opened here. It imports only
|
|
155
|
+
#: the standard library.
|
|
156
|
+
_PLANNER_COMMAND = [sys.executable,
|
|
157
|
+
os.path.join(os.path.dirname(os.path.abspath(
|
|
158
|
+
_planner.__file__)), "_lib_reclaim_planner.py")]
|
|
159
|
+
#: The helper's time budget when a chunk runs outside an attempt; inside
|
|
160
|
+
#: one it is the rest of the attempt's 2 s secondary deadline.
|
|
161
|
+
_PLANNER_DEADLINE_SECONDS = 1.5
|
|
162
|
+
_PLANNER_MIN_DEADLINE_SECONDS = 0.05
|
|
163
|
+
#: Interpreter start-up allowance on top of the helper's own budget.
|
|
164
|
+
_PLANNER_STARTUP_SLACK_SECONDS = 1.0
|
|
165
|
+
#: The helper's read budget in bytes (its time budget is the deadline).
|
|
166
|
+
_PLANNER_READ_BUDGET_BYTES = 256 * MiB
|
|
167
|
+
|
|
168
|
+
#: Mirror of `_cctally_store.STORE_POLICY["conversations"]`'s connection
|
|
169
|
+
#: settings; `tests/test_retention_pacing.py` pins the equality.
|
|
170
|
+
_CONNECTION_BUSY_TIMEOUT_MS = 15_000
|
|
171
|
+
_CONNECTION_SYNCHRONOUS = "NORMAL"
|
|
172
|
+
_CONNECTION_JOURNAL_SIZE_LIMIT = 128 * MiB
|
|
166
173
|
|
|
167
174
|
|
|
168
175
|
@dataclass(frozen=True)
|
|
169
176
|
class PruneStats:
|
|
170
|
-
"""Counts from one prune pass.
|
|
177
|
+
"""Counts from one prune pass. ``complete`` is False when a paced visit
|
|
178
|
+
stopped before every expired group was deleted (a continuation)."""
|
|
171
179
|
|
|
172
180
|
claude_sessions: int = 0
|
|
173
181
|
claude_messages: int = 0
|
|
174
182
|
codex_conversations: int = 0
|
|
175
183
|
codex_events: int = 0
|
|
184
|
+
complete: bool = True
|
|
176
185
|
|
|
177
186
|
@property
|
|
178
187
|
def total_rows(self) -> int:
|
|
179
188
|
return self.claude_messages + self.codex_events
|
|
180
189
|
|
|
181
190
|
|
|
191
|
+
def _iso(value: "dt.datetime | None") -> "str | None":
|
|
192
|
+
return None if value is None else _budget.iso_z(value)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _ceil_second(value: dt.datetime) -> dt.datetime:
|
|
196
|
+
"""``value`` rounded UP to a whole UTC second (idempotent)."""
|
|
197
|
+
value = value.astimezone(UTC)
|
|
198
|
+
if value.microsecond:
|
|
199
|
+
value = value.replace(microsecond=0) + dt.timedelta(seconds=1)
|
|
200
|
+
return value
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _iso_ceil(value: "dt.datetime | None") -> "str | None":
|
|
204
|
+
"""A pacing instant at whole-second precision, rounded UP (901-RW-001).
|
|
205
|
+
|
|
206
|
+
The record's instants are 20-character whole seconds. Truncating the
|
|
207
|
+
allowance's ``as_of`` let every reload of the record at the same
|
|
208
|
+
fractional visit instant accrue the dropped fraction again (I4), and
|
|
209
|
+
truncating ``next_attempt_at`` moved the reclaim cadence earlier. Rounding
|
|
210
|
+
up can only withhold accrual and delay a deadline."""
|
|
211
|
+
return None if value is None else _budget.iso_z(_ceil_second(value))
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _parse_utc(value) -> "dt.datetime | None":
|
|
215
|
+
if not isinstance(value, str):
|
|
216
|
+
return None
|
|
217
|
+
try:
|
|
218
|
+
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
219
|
+
except ValueError:
|
|
220
|
+
return None
|
|
221
|
+
if parsed.tzinfo is None:
|
|
222
|
+
parsed = parsed.replace(tzinfo=UTC)
|
|
223
|
+
return parsed.astimezone(UTC)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _int_or_none(value) -> "int | None":
|
|
227
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
228
|
+
return None
|
|
229
|
+
return value
|
|
230
|
+
|
|
231
|
+
|
|
182
232
|
def _cutoff_iso(cutoff_utc: dt.datetime) -> str:
|
|
183
233
|
"""Whole-second UTC ``...Z`` boundary for lex comparison against the stored
|
|
184
|
-
``...Z`` timestamps.
|
|
185
|
-
|
|
186
|
-
Second-granular: the sub-second mixed-precision edge at the exact cutoff
|
|
187
|
-
second is immaterial for a multi-day retention window — any message wrongly classified
|
|
188
|
-
there is re-derivable from JSONL and the boundary self-corrects on the next
|
|
189
|
-
(daily) prune.
|
|
190
|
-
"""
|
|
234
|
+
``...Z`` timestamps (second-granular; immaterial for a multi-day window)."""
|
|
191
235
|
return cutoff_utc.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
192
236
|
|
|
193
237
|
|
|
194
|
-
|
|
195
|
-
conn: sqlite3.Connection,
|
|
196
|
-
*,
|
|
197
|
-
cutoff_utc: dt.datetime,
|
|
198
|
-
after_group: "Callable[[], None] | None" = None,
|
|
199
|
-
) -> PruneStats:
|
|
200
|
-
"""Prune every transcript group whose latest activity is before ``cutoff_utc``.
|
|
238
|
+
# ── pacing state (the extended #780 record) ───────────────────────────────
|
|
201
239
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
)
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
240
|
+
@dataclass(frozen=True)
|
|
241
|
+
class PacingState:
|
|
242
|
+
balance_bytes: int
|
|
243
|
+
as_of: dt.datetime
|
|
244
|
+
ledger: tuple = ()
|
|
245
|
+
eligible: bool = False
|
|
246
|
+
next_attempt_at: "dt.datetime | None" = None
|
|
247
|
+
continuation_cutoff: "str | None" = None
|
|
248
|
+
last_failure: "dict | None" = None
|
|
249
|
+
progress: "dict | None" = None
|
|
250
|
+
op_seq: int = 0
|
|
251
|
+
legacy: dict = dataclasses.field(default_factory=dict)
|
|
252
|
+
|
|
253
|
+
@classmethod
|
|
254
|
+
def from_record(cls, record, now_utc: dt.datetime) -> "PacingState":
|
|
255
|
+
"""A legacy or absent record reads as full credit and no charges."""
|
|
256
|
+
now = now_utc.astimezone(UTC)
|
|
257
|
+
legacy = {}
|
|
258
|
+
if isinstance(record, dict):
|
|
259
|
+
legacy = {k: record[k] for k in _LEGACY_RECORD_KEYS if k in record}
|
|
260
|
+
if not isinstance(record, dict) or record.get("policy_version") is None:
|
|
261
|
+
return cls(balance_bytes=_budget.MAINTENANCE_CREDIT_BYTES,
|
|
262
|
+
as_of=now, legacy=legacy)
|
|
263
|
+
balance = _int_or_none(record.get("balance_bytes"))
|
|
264
|
+
as_of = _parse_utc(record.get("as_of"))
|
|
265
|
+
if balance is None or as_of is None:
|
|
266
|
+
balance, as_of = _budget.MAINTENANCE_CREDIT_BYTES, now
|
|
267
|
+
failure = record.get("last_failure")
|
|
268
|
+
progress = record.get("progress")
|
|
269
|
+
cutoff = record.get("continuation_cutoff")
|
|
270
|
+
return cls(
|
|
271
|
+
balance_bytes=min(balance, _budget.MAINTENANCE_CREDIT_BYTES),
|
|
272
|
+
as_of=as_of,
|
|
273
|
+
ledger=tuple(_budget.serialize_ledger(
|
|
274
|
+
_budget.parse_ledger(record.get("ledger")))),
|
|
275
|
+
eligible=record.get("eligible") is True,
|
|
276
|
+
next_attempt_at=_parse_utc(record.get("next_attempt_at")),
|
|
277
|
+
continuation_cutoff=cutoff if isinstance(cutoff, str) else None,
|
|
278
|
+
last_failure=failure if isinstance(failure, dict) else None,
|
|
279
|
+
progress=progress if isinstance(progress, dict) else None,
|
|
280
|
+
op_seq=max(0, _int_or_none(record.get("op_seq")) or 0),
|
|
281
|
+
legacy=legacy,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
def available(self, now_utc: dt.datetime) -> int:
|
|
285
|
+
elapsed = max(0.0, (now_utc.astimezone(UTC) - self.as_of).total_seconds())
|
|
286
|
+
accrued = self.balance_bytes + int(
|
|
287
|
+
_budget.MAINTENANCE_RATE_BYTES_PER_MINUTE * elapsed / 60.0)
|
|
288
|
+
return min(_budget.MAINTENANCE_CREDIT_BYTES, accrued)
|
|
289
|
+
|
|
290
|
+
def charge(self, reservation: int, now_utc: dt.datetime, *,
|
|
291
|
+
started_utc: "dt.datetime | None" = None) -> "PacingState":
|
|
292
|
+
"""Subtract the reservation at ``now_utc`` (the accrual instant) and
|
|
293
|
+
add it to the bucket of the hour the operation started,
|
|
294
|
+
``started_utc`` (PR-13 (a); the accrual instant when omitted)."""
|
|
295
|
+
now = now_utc.astimezone(UTC)
|
|
296
|
+
start = now if started_utc is None else started_utc.astimezone(UTC)
|
|
297
|
+
return dataclasses.replace(
|
|
298
|
+
self,
|
|
299
|
+
balance_bytes=self.available(now) - int(reservation),
|
|
300
|
+
as_of=max(now, self.as_of),
|
|
301
|
+
ledger=tuple(_budget.ledger_add(
|
|
302
|
+
list(self.ledger), at_utc=start,
|
|
303
|
+
charge_bytes=int(reservation))),
|
|
304
|
+
op_seq=self.op_seq + 1,
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
def to_record(self) -> dict:
|
|
308
|
+
record = dict(self.legacy)
|
|
309
|
+
record.update({
|
|
310
|
+
"policy_version": RETENTION_POLICY_VERSION,
|
|
311
|
+
"balance_bytes": int(self.balance_bytes),
|
|
312
|
+
"as_of": _iso_ceil(self.as_of),
|
|
313
|
+
"ledger": list(self.ledger),
|
|
314
|
+
"eligible": bool(self.eligible),
|
|
315
|
+
"next_attempt_at": _iso_ceil(self.next_attempt_at),
|
|
316
|
+
"continuation_cutoff": self.continuation_cutoff,
|
|
317
|
+
"last_failure": self.last_failure,
|
|
318
|
+
"progress": self.progress,
|
|
319
|
+
"op_seq": int(self.op_seq),
|
|
320
|
+
})
|
|
321
|
+
return record
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _operation_start(visit_at: dt.datetime, op_clock: float,
|
|
325
|
+
visit_clock0: "float | None") -> dt.datetime:
|
|
326
|
+
"""The operation's start: the visit instant plus the visit clock's
|
|
327
|
+
elapsed time (never negative), or the visit instant itself."""
|
|
328
|
+
if visit_clock0 is None:
|
|
329
|
+
return visit_at
|
|
330
|
+
return visit_at + dt.timedelta(seconds=max(0.0, op_clock - visit_clock0))
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
# ── the fixed-size record (spec §5.4 "State write", 901-SR-016) ──────────
|
|
334
|
+
#
|
|
335
|
+
# Every writer stores the record as `json.dumps(record, sort_keys=True)`
|
|
336
|
+
# space-padded to exactly `RECLAIM_RECORD_LENGTH` (L) bytes, so a reclaim
|
|
337
|
+
# chunk's same-key `UPDATE` replaces the cell by one of the same size: SQLite
|
|
338
|
+
# overwrites it in place (`sqlite3BtreeInsert`'s same-size path, audited at
|
|
339
|
+
# 3.37.2 and 3.53.4), dirtying exactly the one leaf that holds the row, with
|
|
340
|
+
# no rebalance and no page allocated or freed. No writer deletes the row.
|
|
341
|
+
#
|
|
342
|
+
# L is chosen from the record's BOUNDED fields (`_bounded_record` clamps and
|
|
343
|
+
# drops everything else): `max_record_length()` serializes the worst case -
|
|
344
|
+
# every integer at its widest, every instant at 20 characters, a 48-character
|
|
345
|
+
# failure reason and 25 ledger buckets - which is 3,101 bytes. L = 3,968 is
|
|
346
|
+
# 1.279 x that (at least 25% headroom), and the whole cell stays local at
|
|
347
|
+
# 4 KiB pages: a 4-byte record header + the 38-byte key + L = 4,010 bytes of
|
|
348
|
+
# payload <= U - 35 = 4,061 (no overflow page). Both are asserted at import.
|
|
349
|
+
RECLAIM_RECORD_LENGTH = 3968
|
|
350
|
+
_RECORD_HEADROOM_NUMERATOR, _RECORD_HEADROOM_DENOMINATOR = 5, 4
|
|
351
|
+
_INT64_MIN, _INT64_MAX = -(2 ** 63), 2 ** 63 - 1
|
|
352
|
+
_POLICY_VERSION_MAX = 999
|
|
353
|
+
_REASON_MAX_CHARS = 48
|
|
354
|
+
_LEDGER_KEEP = 25
|
|
355
|
+
_NONNEGATIVE_INT_KEYS = ("freelist_count", "unreclaimed_bytes", "op_seq")
|
|
356
|
+
_INSTANT_KEYS = ("attempted_at", "as_of", "next_attempt_at",
|
|
357
|
+
"continuation_cutoff")
|
|
358
|
+
#: The pacing instants, rounded UP to a whole second (901-RW-001); the
|
|
359
|
+
#: others are observations and keep the truncated form.
|
|
360
|
+
_CEIL_INSTANT_KEYS = ("as_of", "next_attempt_at")
|
|
361
|
+
_BOOL_KEYS = ("made_progress", "deadline_hit", "eligible")
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
class StateRecordOversize(ValueError):
|
|
365
|
+
"""A serialized record longer than `RECLAIM_RECORD_LENGTH`."""
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _clamp_int(value, low=_INT64_MIN, high=_INT64_MAX) -> "int | None":
|
|
369
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
370
|
+
return None
|
|
371
|
+
return max(low, min(high, value))
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _bounded_instant(value, *, ceil: bool = False) -> "str | None":
|
|
375
|
+
parsed = _parse_utc(value)
|
|
376
|
+
if parsed is not None and ceil:
|
|
377
|
+
try:
|
|
378
|
+
parsed = _ceil_second(parsed)
|
|
379
|
+
except OverflowError:
|
|
380
|
+
return None
|
|
381
|
+
if parsed is None or not 1000 <= parsed.year <= 9999:
|
|
382
|
+
return None
|
|
383
|
+
return _budget.iso_z(parsed)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _bounded_reason(value) -> str:
|
|
387
|
+
if (isinstance(value, str) and 0 < len(value) <= _REASON_MAX_CHARS
|
|
388
|
+
and all(c.islower() or c.isdigit() or c == "_" for c in value)
|
|
389
|
+
and value.isascii()):
|
|
390
|
+
return value
|
|
391
|
+
return "unknown"
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _bounded_record(record: dict) -> dict:
|
|
395
|
+
"""The record restricted to its known, bounded fields: what every writer
|
|
396
|
+
stores, so the serialized length can never exceed the computed maximum."""
|
|
397
|
+
out = {}
|
|
398
|
+
for key in _NONNEGATIVE_INT_KEYS:
|
|
399
|
+
if key in record:
|
|
400
|
+
out[key] = _clamp_int(record[key], 0)
|
|
401
|
+
if "balance_bytes" in record:
|
|
402
|
+
out["balance_bytes"] = _clamp_int(record["balance_bytes"])
|
|
403
|
+
if "policy_version" in record:
|
|
404
|
+
out["policy_version"] = _clamp_int(record["policy_version"], 0,
|
|
405
|
+
_POLICY_VERSION_MAX)
|
|
406
|
+
for key in _INSTANT_KEYS:
|
|
407
|
+
if key in record:
|
|
408
|
+
out[key] = _bounded_instant(record[key],
|
|
409
|
+
ceil=key in _CEIL_INSTANT_KEYS)
|
|
410
|
+
for key in _BOOL_KEYS:
|
|
411
|
+
if key in record:
|
|
412
|
+
out[key] = record[key] is True
|
|
413
|
+
if "ledger" in record:
|
|
414
|
+
buckets = _budget.parse_ledger(record["ledger"])[-_LEDGER_KEEP:]
|
|
415
|
+
out["ledger"] = [
|
|
416
|
+
{"hour": _budget.iso_z(b.hour),
|
|
417
|
+
"charged": _clamp_int(b.charged, 0),
|
|
418
|
+
"largest": _clamp_int(b.largest, 0)} for b in buckets]
|
|
419
|
+
if "last_failure" in record:
|
|
420
|
+
failure = record["last_failure"]
|
|
421
|
+
out["last_failure"] = (
|
|
422
|
+
{"reason": _bounded_reason(failure.get("reason")),
|
|
423
|
+
"at": _bounded_instant(failure.get("at"))}
|
|
424
|
+
if isinstance(failure, dict) else None)
|
|
425
|
+
if "progress" in record:
|
|
426
|
+
progress = record["progress"]
|
|
427
|
+
out["progress"] = (
|
|
428
|
+
{"freelist_pages": _clamp_int(progress.get("freelist_pages"), 0),
|
|
429
|
+
"page_size": _clamp_int(progress.get("page_size"), 0),
|
|
430
|
+
"observed_at": _bounded_instant(progress.get("observed_at"))}
|
|
431
|
+
if isinstance(progress, dict) else None)
|
|
432
|
+
return out
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _record_json(record: dict) -> str:
|
|
436
|
+
return json.dumps(_bounded_record(record), sort_keys=True)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def serialize_reclaim_record(record: dict) -> str:
|
|
440
|
+
"""The record as exactly `RECLAIM_RECORD_LENGTH` bytes of space-padded
|
|
441
|
+
JSON; raises `StateRecordOversize` rather than store another length."""
|
|
442
|
+
text = _record_json(record)
|
|
443
|
+
if len(text.encode("utf-8")) > RECLAIM_RECORD_LENGTH:
|
|
444
|
+
raise StateRecordOversize(len(text))
|
|
445
|
+
return text + " " * (RECLAIM_RECORD_LENGTH - len(text.encode("utf-8")))
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def max_record_length() -> int:
|
|
449
|
+
"""The longest record `_bounded_record` can produce."""
|
|
450
|
+
instant = "9999-12-31T23:59:59Z"
|
|
451
|
+
widest_nonnegative = _INT64_MAX
|
|
452
|
+
worst = {key: widest_nonnegative for key in _NONNEGATIVE_INT_KEYS}
|
|
453
|
+
worst.update({key: instant for key in _INSTANT_KEYS})
|
|
454
|
+
worst.update({key: False for key in _BOOL_KEYS})
|
|
455
|
+
worst.update({
|
|
456
|
+
"balance_bytes": _INT64_MIN,
|
|
457
|
+
"policy_version": _POLICY_VERSION_MAX,
|
|
458
|
+
"ledger": [{"hour": f"9999-12-{1 + h // 24:02d}T{h % 24:02d}:00:00Z",
|
|
459
|
+
"charged": widest_nonnegative,
|
|
460
|
+
"largest": widest_nonnegative}
|
|
461
|
+
for h in range(_LEDGER_KEEP)],
|
|
462
|
+
"last_failure": {"reason": "x" * _REASON_MAX_CHARS, "at": instant},
|
|
463
|
+
"progress": {"freelist_pages": widest_nonnegative,
|
|
464
|
+
"page_size": widest_nonnegative, "observed_at": instant},
|
|
465
|
+
})
|
|
466
|
+
return len(_record_json(worst).encode("utf-8"))
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _varint_size(value: int) -> int:
|
|
470
|
+
size = 1
|
|
471
|
+
while value >= 0x80 and size < 9:
|
|
472
|
+
value >>= 7
|
|
473
|
+
size += 1
|
|
474
|
+
return size
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def record_cell_payload_bytes(length: int = RECLAIM_RECORD_LENGTH) -> int:
|
|
478
|
+
"""The `cache_meta(key, value)` record payload of the #780 row: header
|
|
479
|
+
(its size varint + the key's and the value's serial types), key, value."""
|
|
480
|
+
key = len(RECLAIM_PENDING_KEY.encode("utf-8"))
|
|
481
|
+
types = _varint_size(2 * key + 13) + _varint_size(2 * length + 13)
|
|
482
|
+
header = types + 1
|
|
483
|
+
return _varint_size(header) - 1 + header + key + length
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def record_fits_locally(usable_size: int,
|
|
487
|
+
length: int = RECLAIM_RECORD_LENGTH) -> bool:
|
|
488
|
+
"""A table-leaf cell keeps its whole payload local up to U - 35 bytes."""
|
|
489
|
+
return record_cell_payload_bytes(length) <= int(usable_size) - 35
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
assert RECLAIM_RECORD_LENGTH * _RECORD_HEADROOM_DENOMINATOR >= (
|
|
493
|
+
max_record_length() * _RECORD_HEADROOM_NUMERATOR), (
|
|
494
|
+
"the #780 record length needs 25% headroom over its largest record")
|
|
495
|
+
assert record_fits_locally(4096), (
|
|
496
|
+
"the #780 record cell must stay local on a 4 KiB page")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def read_reclaim_pending(conn: sqlite3.Connection) -> "dict | None":
|
|
500
|
+
"""The durable #780/#901 record (padding stripped), or None when absent
|
|
501
|
+
or unreadable."""
|
|
502
|
+
try:
|
|
503
|
+
row = conn.execute(
|
|
504
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
505
|
+
(RECLAIM_PENDING_KEY,),
|
|
506
|
+
).fetchone()
|
|
507
|
+
except sqlite3.Error:
|
|
508
|
+
return None
|
|
509
|
+
if row is None or not row[0]:
|
|
510
|
+
return None
|
|
511
|
+
try:
|
|
512
|
+
state = json.loads(str(row[0]).rstrip(" "))
|
|
513
|
+
except (TypeError, ValueError):
|
|
514
|
+
return None
|
|
515
|
+
return state if isinstance(state, dict) else None
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def reclaim_record_length(conn: sqlite3.Connection) -> "int | None":
|
|
519
|
+
"""The stored record's length in bytes, or None when the row is absent."""
|
|
520
|
+
row = conn.execute(
|
|
521
|
+
"SELECT length(CAST(value AS BLOB)) FROM cache_meta WHERE key=?",
|
|
522
|
+
(RECLAIM_PENDING_KEY,),
|
|
523
|
+
).fetchone()
|
|
524
|
+
return None if row is None or row[0] is None else int(row[0])
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _write_reclaim_pending(conn: sqlite3.Connection, state: "dict | None") -> None:
|
|
528
|
+
"""Store the record at exactly L bytes. Nothing deletes the row: `None`
|
|
529
|
+
(an ended episode) rewrites the current record with `eligible` false."""
|
|
530
|
+
if state is None:
|
|
531
|
+
current = read_reclaim_pending(conn) or {}
|
|
532
|
+
state = {**current, "eligible": False}
|
|
533
|
+
conn.execute(
|
|
534
|
+
"INSERT INTO cache_meta(key, value) VALUES (?, ?) "
|
|
535
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
536
|
+
(RECLAIM_PENDING_KEY, serialize_reclaim_record(state)),
|
|
219
537
|
)
|
|
220
538
|
|
|
221
539
|
|
|
540
|
+
def _write_reclaim_state_in_chunk(conn: sqlite3.Connection,
|
|
541
|
+
record: dict) -> None:
|
|
542
|
+
"""A reclaim chunk's state write: a same-key `UPDATE` of the existing
|
|
543
|
+
L-byte row, never an insert or `REPLACE`, so it overwrites one cell in
|
|
544
|
+
place. Raises `StateRecordOversize` before writing anything."""
|
|
545
|
+
value = serialize_reclaim_record(record)
|
|
546
|
+
cursor = conn.execute("UPDATE cache_meta SET value=? WHERE key=?",
|
|
547
|
+
(value, RECLAIM_PENDING_KEY))
|
|
548
|
+
if cursor.rowcount != 1:
|
|
549
|
+
raise sqlite3.OperationalError("the #780 record row is missing")
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def normalize_reclaim_record(conn: sqlite3.Connection,
|
|
553
|
+
now_utc: dt.datetime) -> bool:
|
|
554
|
+
"""Write the record at length L when it is absent (an older binary's
|
|
555
|
+
deleted-row state) or of another length (a legacy or pre-#901 record),
|
|
556
|
+
in its own state-only transaction outside any chunk. Uncharged, and
|
|
557
|
+
measured under I2 like every other maintenance state write. Returns True
|
|
558
|
+
when it wrote. `conn` must hold no open transaction."""
|
|
559
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
560
|
+
try:
|
|
561
|
+
if reclaim_record_length(conn) == RECLAIM_RECORD_LENGTH:
|
|
562
|
+
conn.rollback()
|
|
563
|
+
return False
|
|
564
|
+
record = read_reclaim_pending(conn)
|
|
565
|
+
if not isinstance(record, dict) or record.get("policy_version") is None:
|
|
566
|
+
record = PacingState.from_record(record, now_utc).to_record()
|
|
567
|
+
_write_reclaim_pending(conn, record)
|
|
568
|
+
conn.commit()
|
|
569
|
+
return True
|
|
570
|
+
except BaseException:
|
|
571
|
+
conn.rollback()
|
|
572
|
+
raise
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _ceil_div(numerator: int, denominator: int) -> int:
|
|
576
|
+
return -(-int(numerator) // int(denominator))
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def page_frame_bytes(page_size: int) -> int:
|
|
580
|
+
"""One distinct dirty page's whole cost: its WAL frame (page + 24-byte
|
|
581
|
+
header) and its eventual checkpoint copy (spec I4)."""
|
|
582
|
+
return 2 * int(page_size) + 24
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def pointer_map_page_bytes(page_size: int) -> int:
|
|
586
|
+
"""B: 1.25 x (2P + 24) rounded up to a KiB, from the store's page size."""
|
|
587
|
+
return _ceil_div(POINTER_MAP_FACTOR_NUMERATOR * page_frame_bytes(page_size),
|
|
588
|
+
POINTER_MAP_FACTOR_DENOMINATOR * KiB) * KiB
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def pointer_map_cap(page_count: int, usable_size: int, rows: int,
|
|
592
|
+
page_size: int) -> int:
|
|
593
|
+
"""I4's M_cap = ceil(N_max / (floor(U/5) + 1)) + 1, where N_max is the
|
|
594
|
+
page count at the operation's start plus ceil((512 KiB + 12 KiB x rows)
|
|
595
|
+
/ P): every other dirty page appended at the end of the file (Q8)."""
|
|
596
|
+
growth = _ceil_div(I4_DELETION_FIXED_BYTES
|
|
597
|
+
+ I4_DELETION_PER_ROW_BYTES * max(0, int(rows)),
|
|
598
|
+
int(page_size))
|
|
599
|
+
n_max = int(page_count) + growth
|
|
600
|
+
return _ceil_div(n_max, int(usable_size) // 5 + 1) + 1
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def deletion_reservation(rows: int, *, page_count: int, usable_size: int,
|
|
604
|
+
page_size: int) -> int:
|
|
605
|
+
"""F + A x rows + B x M_cap (spec §5.4, Q8), from geometry read inside
|
|
606
|
+
the operation's `BEGIN IMMEDIATE`."""
|
|
607
|
+
rows = max(0, int(rows))
|
|
608
|
+
per_row = (DELETION_PER_ROW_FALLBACK_BYTES
|
|
609
|
+
if rows > DELETION_FALLBACK_ROW_THRESHOLD
|
|
610
|
+
else DELETION_PER_ROW_BYTES)
|
|
611
|
+
return (DELETION_FIXED_BYTES + per_row * rows
|
|
612
|
+
+ pointer_map_page_bytes(page_size)
|
|
613
|
+
* pointer_map_cap(page_count, usable_size, rows, page_size))
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def _record_int(op: dict, key: str) -> "int | None":
|
|
617
|
+
value = op.get(key)
|
|
618
|
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
|
619
|
+
return None
|
|
620
|
+
return value
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def recompute_charge(op: dict) -> "int | None":
|
|
624
|
+
"""The reservation an operation record's own inputs imply, or None when
|
|
625
|
+
the record lacks them or carries another formula version. `op` is an
|
|
626
|
+
operation payload (`as_phase_payload`) or a published maintenance record
|
|
627
|
+
with its `phase` ("delete" | "reclaim")."""
|
|
628
|
+
if not isinstance(op, dict):
|
|
629
|
+
return None
|
|
630
|
+
if op.get("reservation_version") != RESERVATION_VERSION:
|
|
631
|
+
return None
|
|
632
|
+
phase = op.get("phase") or op.get("kind")
|
|
633
|
+
if phase == "delete":
|
|
634
|
+
values = [_record_int(op, k) for k in (
|
|
635
|
+
"rows", "page_count", "usable_size", "page_size")]
|
|
636
|
+
if None in values or values[3] <= 0 or values[2] <= 0:
|
|
637
|
+
return None
|
|
638
|
+
rows, page_count, usable_size, page_size = values
|
|
639
|
+
return deletion_reservation(rows, page_count=page_count,
|
|
640
|
+
usable_size=usable_size,
|
|
641
|
+
page_size=page_size)
|
|
642
|
+
if phase == "reclaim":
|
|
643
|
+
values = [_record_int(op, k) for k in (
|
|
644
|
+
"identified_pages", "unidentified_pages", "page_size",
|
|
645
|
+
"fixed_bytes")]
|
|
646
|
+
if None in values or values[2] <= 0 or values[0] < 1:
|
|
647
|
+
return None
|
|
648
|
+
identified, unidentified, page_size, fixed = values
|
|
649
|
+
return reclaim_reservation(identified, unidentified,
|
|
650
|
+
page_size=page_size, fixed_bytes=fixed)
|
|
651
|
+
return None
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
def reclaim_reservation(identified: int, unidentified: int, *,
|
|
655
|
+
page_size: int,
|
|
656
|
+
fixed_bytes: int = RECLAIM_FIXED_BYTES) -> int:
|
|
657
|
+
"""(|ID| + UNID + 1) x (2P + 24) + F_r from a chunk's plan (Q9)."""
|
|
658
|
+
return _planner.reservation_bytes(identified, unidentified, page_size,
|
|
659
|
+
fixed_bytes)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
# ── store geometry, eligibility and the ceiling ──────────────────────────
|
|
663
|
+
|
|
664
|
+
@dataclass(frozen=True)
|
|
665
|
+
class StoreGeometry:
|
|
666
|
+
page_count: int
|
|
667
|
+
freelist_count: int
|
|
668
|
+
page_size: int
|
|
669
|
+
|
|
670
|
+
@property
|
|
671
|
+
def free_bytes(self) -> int:
|
|
672
|
+
return self.freelist_count * self.page_size
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
#: `SQLITE_FCNTL_RESERVE_BYTES` (sqlite3.h): a negative argument only reads
|
|
676
|
+
#: the bytes reserved at the end of every page.
|
|
677
|
+
_SQLITE_FCNTL_RESERVE_BYTES = 38
|
|
678
|
+
#: The most a page can reserve: the conservative usable size when the
|
|
679
|
+
#: reserve cannot be read (a smaller U only raises M_cap and the charge).
|
|
680
|
+
_MAX_RESERVED_BYTES = 255
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _sqlite_handle(conn) -> "int | None":
|
|
684
|
+
"""The `sqlite3 *` behind a CPython connection (through test wrappers
|
|
685
|
+
that delegate via `_conn`), as `bin/_lib_sqlite_close.py` reaches it."""
|
|
686
|
+
real = conn
|
|
687
|
+
for _ in range(4):
|
|
688
|
+
if isinstance(real, sqlite3.Connection):
|
|
689
|
+
break
|
|
690
|
+
real = getattr(real, "_conn", None)
|
|
691
|
+
if (not isinstance(real, sqlite3.Connection)
|
|
692
|
+
or sys.implementation.name != "cpython"):
|
|
693
|
+
return None
|
|
694
|
+
import ctypes
|
|
695
|
+
|
|
696
|
+
return ctypes.c_void_p.from_address(
|
|
697
|
+
id(real) + 2 * ctypes.sizeof(ctypes.c_void_p)).value or None
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def reserved_bytes(conn) -> "int | None":
|
|
701
|
+
"""Bytes reserved per page of `main`, read through the stdlib
|
|
702
|
+
extension's own SQLite (never a raw descriptor of the database file,
|
|
703
|
+
whose close would drop this process's POSIX locks); None when it cannot
|
|
704
|
+
be read."""
|
|
705
|
+
handle = _sqlite_handle(conn)
|
|
706
|
+
if handle is None:
|
|
707
|
+
return None
|
|
708
|
+
try:
|
|
709
|
+
import _sqlite3
|
|
710
|
+
import ctypes
|
|
711
|
+
|
|
712
|
+
file_control = ctypes.CDLL(_sqlite3.__file__).sqlite3_file_control
|
|
713
|
+
file_control.argtypes = (ctypes.c_void_p, ctypes.c_char_p,
|
|
714
|
+
ctypes.c_int, ctypes.c_void_p)
|
|
715
|
+
file_control.restype = ctypes.c_int
|
|
716
|
+
value = ctypes.c_int(-1)
|
|
717
|
+
rc = file_control(ctypes.c_void_p(handle), b"main",
|
|
718
|
+
_SQLITE_FCNTL_RESERVE_BYTES, ctypes.byref(value))
|
|
719
|
+
except (OSError, AttributeError, ImportError):
|
|
720
|
+
return None
|
|
721
|
+
if rc != 0 or not 0 <= value.value <= _MAX_RESERVED_BYTES:
|
|
722
|
+
return None
|
|
723
|
+
return int(value.value)
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
@dataclass(frozen=True)
|
|
727
|
+
class DeletionGeometry:
|
|
728
|
+
"""The Q8 reservation inputs, read inside the operation's transaction."""
|
|
729
|
+
page_count: int
|
|
730
|
+
page_size: int
|
|
731
|
+
usable_size: int
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def deletion_geometry(conn: sqlite3.Connection) -> DeletionGeometry:
|
|
735
|
+
page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
|
|
736
|
+
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
|
|
737
|
+
reserved = reserved_bytes(conn)
|
|
738
|
+
usable = page_size - (_MAX_RESERVED_BYTES if reserved is None else reserved)
|
|
739
|
+
return DeletionGeometry(page_count, page_size, usable)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def store_geometry(conn: sqlite3.Connection) -> "StoreGeometry | None":
|
|
743
|
+
try:
|
|
744
|
+
page_count = int(conn.execute("PRAGMA page_count").fetchone()[0])
|
|
745
|
+
freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
|
746
|
+
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
|
|
747
|
+
except (sqlite3.Error, TypeError, ValueError):
|
|
748
|
+
return None
|
|
749
|
+
if page_count <= 0 or page_size <= 0 or freelist < 0:
|
|
750
|
+
return None
|
|
751
|
+
return StoreGeometry(page_count, freelist, page_size)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def reclaim_episode_active(geometry: "StoreGeometry | None",
|
|
755
|
+
episode_active: bool) -> bool:
|
|
756
|
+
"""Start above 2 GiB AND 20%; an active episode stops at 1 GiB OR 10%."""
|
|
757
|
+
if geometry is None:
|
|
758
|
+
return False
|
|
759
|
+
free = geometry.free_bytes
|
|
760
|
+
ratio = geometry.freelist_count / geometry.page_count
|
|
761
|
+
if episode_active:
|
|
762
|
+
return free > RECLAIM_STOP_BYTES and ratio > RECLAIM_STOP_RATIO
|
|
763
|
+
return free > RECLAIM_START_BYTES and ratio > RECLAIM_START_RATIO
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def observed_reclaim_backlog_bytes(conn: sqlite3.Connection) -> "int | None":
|
|
767
|
+
geometry = store_geometry(conn)
|
|
768
|
+
return None if geometry is None else geometry.free_bytes
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def backlog_over_ceiling(backlog_bytes: "int | None") -> bool:
|
|
772
|
+
return backlog_bytes is not None and backlog_bytes >= RECLAIM_CEILING_BYTES
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def observed_backlog_over_ceiling(conn: sqlite3.Connection) -> bool:
|
|
776
|
+
"""Doctor FAIL and the rebuild refusal: the OBSERVED freelist."""
|
|
777
|
+
return backlog_over_ceiling(observed_reclaim_backlog_bytes(conn))
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
# ── connections ───────────────────────────────────────────────────────────
|
|
781
|
+
|
|
782
|
+
def _apply_connection_settings(conn: sqlite3.Connection) -> None:
|
|
783
|
+
conn.execute(f"PRAGMA busy_timeout={_CONNECTION_BUSY_TIMEOUT_MS}")
|
|
784
|
+
conn.execute(f"PRAGMA synchronous={_CONNECTION_SYNCHRONOUS}")
|
|
785
|
+
conn.execute(f"PRAGMA journal_size_limit={_CONNECTION_JOURNAL_SIZE_LIMIT}")
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def open_deletion_connection(db_path) -> sqlite3.Connection:
|
|
789
|
+
"""The per-visit deletion connection: 256 MiB cache, in-memory temp
|
|
790
|
+
store, SQLite's automatic checkpoint and default close. Closed at the end
|
|
791
|
+
of the visit, so the enlarged cache never outlives it."""
|
|
792
|
+
conn = sqlite3.connect(str(db_path),
|
|
793
|
+
timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
|
|
794
|
+
try:
|
|
795
|
+
_apply_connection_settings(conn)
|
|
796
|
+
conn.execute(f"PRAGMA cache_size = -{DELETION_CACHE_SIZE_KIB}")
|
|
797
|
+
conn.execute("PRAGMA temp_store = MEMORY")
|
|
798
|
+
except BaseException:
|
|
799
|
+
conn.close()
|
|
800
|
+
raise
|
|
801
|
+
return conn
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
class ReclaimUnavailable(Exception):
|
|
805
|
+
"""Reclaim cannot run without checkpointing; carries a typed reason."""
|
|
806
|
+
|
|
807
|
+
def __init__(self, reason: str):
|
|
808
|
+
super().__init__(reason)
|
|
809
|
+
self.reason = reason
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def open_reclaim_connection(db_path) -> sqlite3.Connection:
|
|
813
|
+
"""A connection that never checkpoints: not at a commit
|
|
814
|
+
(`wal_autocheckpoint = 0`) and not at its close, even as the last
|
|
815
|
+
connection (no-checkpoint-on-close). Raises `ReclaimUnavailable` when the
|
|
816
|
+
close setting cannot be applied.
|
|
817
|
+
|
|
818
|
+
Spec §5.4 (revision 8): `cache_spill = OFF` and `temp_store = MEMORY`, so
|
|
819
|
+
no page of a chunk reaches the WAL before its commit and no statement
|
|
820
|
+
journal touches disk."""
|
|
821
|
+
conn = sqlite3.connect(str(db_path),
|
|
822
|
+
timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
|
|
823
|
+
try:
|
|
824
|
+
try:
|
|
825
|
+
_apply_connection_settings(conn)
|
|
826
|
+
conn.execute("PRAGMA cache_spill = OFF")
|
|
827
|
+
conn.execute("PRAGMA temp_store = MEMORY")
|
|
828
|
+
conn.execute("PRAGMA wal_autocheckpoint = 0")
|
|
829
|
+
applied = (
|
|
830
|
+
int(conn.execute("PRAGMA cache_spill").fetchone()[0]),
|
|
831
|
+
int(conn.execute("PRAGMA temp_store").fetchone()[0]))
|
|
832
|
+
except sqlite3.Error as exc:
|
|
833
|
+
# The primary's disposition 5: a pragma that fails is a typed
|
|
834
|
+
# refusal (§5.4 "a failure to apply the reclaim connection's
|
|
835
|
+
# settings"), never a raw sqlite3.Error from the opener.
|
|
836
|
+
raise ReclaimUnavailable("connection_settings_unavailable") from exc
|
|
837
|
+
if applied != (0, 2):
|
|
838
|
+
raise ReclaimUnavailable("connection_settings_unavailable")
|
|
839
|
+
if int(conn.execute("PRAGMA wal_autocheckpoint").fetchone()[0]) != 0:
|
|
840
|
+
raise ReclaimUnavailable("no_checkpoint_on_close_unavailable")
|
|
841
|
+
try:
|
|
842
|
+
_lib_sqlite_close.set_no_checkpoint_on_close(
|
|
843
|
+
conn, True, purpose="paced reclaim")
|
|
844
|
+
except (sqlite3.Error, OSError, AttributeError, TypeError) as exc:
|
|
845
|
+
raise ReclaimUnavailable(
|
|
846
|
+
"no_checkpoint_on_close_unavailable") from exc
|
|
847
|
+
except BaseException:
|
|
848
|
+
conn.close()
|
|
849
|
+
raise
|
|
850
|
+
return conn
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
def _open_state_connection(db_path) -> sqlite3.Connection:
|
|
854
|
+
"""For a state-only record write (episode end, failure): never
|
|
855
|
+
checkpoints at commit; no-checkpoint-on-close where available."""
|
|
856
|
+
conn = sqlite3.connect(str(db_path),
|
|
857
|
+
timeout=_CONNECTION_BUSY_TIMEOUT_MS / 1000)
|
|
858
|
+
_apply_connection_settings(conn)
|
|
859
|
+
conn.execute("PRAGMA wal_autocheckpoint = 0")
|
|
860
|
+
try:
|
|
861
|
+
_lib_sqlite_close.set_no_checkpoint_on_close(
|
|
862
|
+
conn, True, purpose="retention state")
|
|
863
|
+
except (sqlite3.Error, OSError, AttributeError, TypeError):
|
|
864
|
+
pass
|
|
865
|
+
return conn
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
# Seams the tests replace to observe each connection a visit opens.
|
|
869
|
+
_open_deletion_connection = open_deletion_connection
|
|
870
|
+
_open_reclaim_connection = open_reclaim_connection
|
|
871
|
+
|
|
872
|
+
|
|
873
|
+
# ── expired groups ────────────────────────────────────────────────────────
|
|
874
|
+
|
|
222
875
|
def _prunable_groups(
|
|
223
876
|
conn: sqlite3.Connection, table: str, key_col: str, cutoff: str
|
|
224
877
|
) -> list[str]:
|
|
225
|
-
"""
|
|
226
|
-
|
|
227
|
-
(conservative — never pruned in isolation, F12). MAX(...) < cutoff is
|
|
228
|
-
equivalent to NOT EXISTS a row at/after the cutoff (NULLs are ignored by
|
|
229
|
-
MAX and never satisfy ``>= cutoff``)."""
|
|
878
|
+
"""Group keys with a dated row all before the cutoff. MAX(timestamp_utc)
|
|
879
|
+
IS NOT NULL excludes all-NULL-timestamp groups (F12)."""
|
|
230
880
|
sql = (
|
|
231
881
|
f"SELECT {key_col} FROM {table} "
|
|
232
882
|
f"WHERE {key_col} IS NOT NULL "
|
|
@@ -239,78 +889,568 @@ def _prunable_groups(
|
|
|
239
889
|
def _prunable_null_identity_paths(
|
|
240
890
|
conn: sqlite3.Connection, table: str, key_col: str, cutoff: str
|
|
241
891
|
) -> list[str]:
|
|
242
|
-
"""F12:
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
892
|
+
"""F12: NULL-identity rows are pruned per source_path.
|
|
893
|
+
|
|
894
|
+
#901 Amendment 19 T1 (spec I1): the per-path ``MAX(timestamp_utc)`` is
|
|
895
|
+
folded in Python over the streamed NULL-identity rows, one entry per
|
|
896
|
+
path, where ``GROUP BY source_path`` sorted every such row through a temp
|
|
897
|
+
b-tree. The class is legacy (current ingest never writes it), but nothing
|
|
898
|
+
bounds how much of it a store retains. Same result: a path qualifies when
|
|
899
|
+
it has a dated row and its latest dated row is before ``cutoff``, in
|
|
900
|
+
ascending path order.
|
|
901
|
+
"""
|
|
902
|
+
latest: dict = {}
|
|
903
|
+
for source_path, stamp in conn.execute(
|
|
904
|
+
f"SELECT source_path, timestamp_utc FROM {table} "
|
|
905
|
+
f"WHERE {key_col} IS NULL"
|
|
906
|
+
):
|
|
907
|
+
if stamp is None:
|
|
908
|
+
continue
|
|
909
|
+
held = latest.get(source_path)
|
|
910
|
+
if held is None or _sqlite_order(stamp) > _sqlite_order(held):
|
|
911
|
+
latest[source_path] = stamp
|
|
912
|
+
bound = _sqlite_order(cutoff)
|
|
913
|
+
return [
|
|
914
|
+
path for path in sorted(latest, key=_sqlite_order)
|
|
915
|
+
if _sqlite_order(latest[path]) < bound
|
|
916
|
+
]
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _sqlite_order(value) -> tuple:
|
|
920
|
+
"""SQLite's cross-type collation for ``MAX``/``<``/``ORDER BY`` on one
|
|
921
|
+
value: NULL < numbers < text (BINARY) < blobs."""
|
|
922
|
+
if value is None:
|
|
923
|
+
return (0, 0)
|
|
924
|
+
if isinstance(value, (int, float)):
|
|
925
|
+
return (1, value)
|
|
926
|
+
if isinstance(value, str):
|
|
927
|
+
return (2, value.encode("utf-8"))
|
|
928
|
+
return (3, bytes(value))
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
@dataclass(frozen=True)
|
|
932
|
+
class ExpiredGroup:
|
|
933
|
+
provider: str # "claude" | "codex"
|
|
934
|
+
by_path: bool # NULL identity, grouped by source_path (F12)
|
|
935
|
+
value: str
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
_GROUP_TABLES = {
|
|
939
|
+
"claude": ("conversation_messages", "session_id"),
|
|
940
|
+
"codex": ("codex_conversation_events", "conversation_key"),
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def expired_groups(conn: sqlite3.Connection, cutoff_iso: str) -> "list[ExpiredGroup]":
|
|
945
|
+
"""Claude groups first, then Codex: the order the visit deletes them."""
|
|
946
|
+
groups: "list[ExpiredGroup]" = []
|
|
947
|
+
for provider in ("claude", "codex"):
|
|
948
|
+
table, key = _GROUP_TABLES[provider]
|
|
949
|
+
groups += [ExpiredGroup(provider, False, value)
|
|
950
|
+
for value in _prunable_groups(conn, table, key, cutoff_iso)]
|
|
951
|
+
groups += [ExpiredGroup(provider, True, value)
|
|
952
|
+
for value in _prunable_null_identity_paths(
|
|
953
|
+
conn, table, key, cutoff_iso)]
|
|
954
|
+
return groups
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def group_row_count(conn: sqlite3.Connection, group: ExpiredGroup) -> int:
|
|
958
|
+
"""Rows the reservation charges: `conversation_messages` for a Claude
|
|
959
|
+
group, `codex_conversation_events` for a Codex one."""
|
|
960
|
+
table, key = _GROUP_TABLES[group.provider]
|
|
961
|
+
if group.by_path:
|
|
962
|
+
sql = f"SELECT COUNT(*) FROM {table} WHERE {key} IS NULL AND source_path = ?"
|
|
963
|
+
else:
|
|
964
|
+
sql = f"SELECT COUNT(*) FROM {table} WHERE {key} = ?"
|
|
965
|
+
return int(conn.execute(sql, (group.value,)).fetchone()[0])
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _rowcount(cursor) -> int:
|
|
969
|
+
return cursor.rowcount if cursor.rowcount and cursor.rowcount > 0 else 0
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _delete_claude_session(conn: sqlite3.Connection, session_id: str) -> int:
|
|
973
|
+
conn.execute("DELETE FROM conversation_file_touches WHERE session_id = ?",
|
|
974
|
+
(session_id,))
|
|
975
|
+
deleted = _rowcount(conn.execute(
|
|
976
|
+
"DELETE FROM conversation_messages WHERE session_id = ?", (session_id,)))
|
|
977
|
+
conn.execute("DELETE FROM conversation_ai_titles WHERE session_id = ?",
|
|
978
|
+
(session_id,))
|
|
979
|
+
conn.execute("DELETE FROM conversation_sessions WHERE session_id = ?",
|
|
980
|
+
(session_id,))
|
|
981
|
+
return deleted
|
|
982
|
+
|
|
983
|
+
|
|
984
|
+
def _delete_claude_null_path(conn: sqlite3.Connection, source_path: str) -> int:
|
|
985
|
+
# NULL-session messages carry no session_id-keyed titles/rollup, and
|
|
986
|
+
# conversation_file_touches requires a NOT NULL session_id, so stray
|
|
987
|
+
# touches are deleted by message_id before the messages themselves.
|
|
988
|
+
ids = [row[0] for row in conn.execute(
|
|
989
|
+
"SELECT id FROM conversation_messages "
|
|
990
|
+
"WHERE session_id IS NULL AND source_path = ?", (source_path,))]
|
|
991
|
+
if ids:
|
|
992
|
+
conn.executemany(
|
|
993
|
+
"DELETE FROM conversation_file_touches WHERE message_id = ?",
|
|
994
|
+
[(i,) for i in ids])
|
|
995
|
+
return _rowcount(conn.execute(
|
|
996
|
+
"DELETE FROM conversation_messages "
|
|
997
|
+
"WHERE session_id IS NULL AND source_path = ?", (source_path,)))
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
def _delete_codex_conversation_derived(
|
|
1001
|
+
conn: sqlite3.Connection, conversation_key: str
|
|
1002
|
+
) -> None:
|
|
1003
|
+
"""#294 S6: drop one pruned conversation's normalized derived rows in the
|
|
1004
|
+
same transaction as its physical events (§3.2 or-delete, §3.4 partial
|
|
1005
|
+
delete riding the per-row FTS trigger, never `'delete-all'`)."""
|
|
1006
|
+
conn.execute(
|
|
1007
|
+
"DELETE FROM codex_conversation_file_touches WHERE conversation_key = ?",
|
|
1008
|
+
(conversation_key,))
|
|
1009
|
+
conn.execute(
|
|
1010
|
+
"DELETE FROM codex_conversation_messages WHERE conversation_key = ?",
|
|
1011
|
+
(conversation_key,))
|
|
1012
|
+
conn.execute(
|
|
1013
|
+
"DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
|
|
1014
|
+
(conversation_key,))
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def _delete_codex_conversation(conn: sqlite3.Connection, key: str) -> int:
|
|
1018
|
+
_delete_codex_conversation_derived(conn, key)
|
|
1019
|
+
return _rowcount(conn.execute(
|
|
1020
|
+
"DELETE FROM codex_conversation_events WHERE conversation_key = ?",
|
|
1021
|
+
(key,)))
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def _delete_codex_null_path(conn: sqlite3.Connection, source_path: str) -> int:
|
|
1025
|
+
# NULL-conversation_key events never gained thread identity, so S6 never
|
|
1026
|
+
# normalized them (§4.1) — there are no derived rows to clean up.
|
|
1027
|
+
return _rowcount(conn.execute(
|
|
1028
|
+
"DELETE FROM codex_conversation_events "
|
|
1029
|
+
"WHERE conversation_key IS NULL AND source_path = ?", (source_path,)))
|
|
251
1030
|
|
|
252
1031
|
|
|
253
|
-
def
|
|
1032
|
+
def _delete_group_rows(conn: sqlite3.Connection, group: ExpiredGroup) -> int:
|
|
1033
|
+
if group.provider == "claude":
|
|
1034
|
+
if group.by_path:
|
|
1035
|
+
return _delete_claude_null_path(conn, group.value)
|
|
1036
|
+
return _delete_claude_session(conn, group.value)
|
|
1037
|
+
if group.by_path:
|
|
1038
|
+
return _delete_codex_null_path(conn, group.value)
|
|
1039
|
+
return _delete_codex_conversation(conn, group.value)
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def prune_conversation_transcripts(
|
|
254
1043
|
conn: sqlite3.Connection,
|
|
255
|
-
cutoff: str,
|
|
256
1044
|
*,
|
|
1045
|
+
cutoff_utc: dt.datetime,
|
|
257
1046
|
after_group: "Callable[[], None] | None" = None,
|
|
258
|
-
) ->
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
conn
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
1047
|
+
) -> PruneStats:
|
|
1048
|
+
"""The unpaced kernel: prune every expired group inside the caller's
|
|
1049
|
+
transaction. Kept for direct kernel callers; the orchestrator below runs
|
|
1050
|
+
each group as a paced, reserved operation instead."""
|
|
1051
|
+
cutoff = _cutoff_iso(cutoff_utc)
|
|
1052
|
+
claude_sessions, claude_messages = _prune_claude(
|
|
1053
|
+
conn, cutoff, after_group=after_group)
|
|
1054
|
+
codex_conversations, codex_events = _prune_codex(
|
|
1055
|
+
conn, cutoff, after_group=after_group)
|
|
1056
|
+
return PruneStats(
|
|
1057
|
+
claude_sessions=claude_sessions,
|
|
1058
|
+
claude_messages=claude_messages,
|
|
1059
|
+
codex_conversations=codex_conversations,
|
|
1060
|
+
codex_events=codex_events,
|
|
1061
|
+
)
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _prune_provider(conn, provider, cutoff, after_group):
|
|
1065
|
+
groups = 0
|
|
1066
|
+
rows = 0
|
|
1067
|
+
table, key = _GROUP_TABLES[provider]
|
|
1068
|
+
for by_path, values in (
|
|
1069
|
+
(False, _prunable_groups(conn, table, key, cutoff)),
|
|
1070
|
+
(True, _prunable_null_identity_paths(conn, table, key, cutoff))):
|
|
1071
|
+
for value in values:
|
|
1072
|
+
rows += _delete_group_rows(conn, ExpiredGroup(provider, by_path, value))
|
|
1073
|
+
groups += 1
|
|
1074
|
+
if after_group is not None:
|
|
1075
|
+
after_group()
|
|
1076
|
+
return groups, rows
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def _prune_claude(conn, cutoff, *, after_group=None) -> "tuple[int, int]":
|
|
1080
|
+
return _prune_provider(conn, "claude", cutoff, after_group)
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _prune_codex(conn, cutoff, *, after_group=None) -> "tuple[int, int]":
|
|
1084
|
+
return _prune_provider(conn, "codex", cutoff, after_group)
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
# ── operations (spec §5.4 protocol) ───────────────────────────────────────
|
|
1088
|
+
|
|
1089
|
+
@dataclass(frozen=True)
|
|
1090
|
+
class OperationResult:
|
|
1091
|
+
op_id: int
|
|
1092
|
+
kind: str # "delete" | "reclaim"
|
|
1093
|
+
outcome: str # "ok" | "gated" | "no_progress" | "skipped"
|
|
1094
|
+
started_at: dt.datetime
|
|
1095
|
+
duration_s: float
|
|
1096
|
+
rows: int = 0
|
|
1097
|
+
pages: int = 0
|
|
1098
|
+
charged_bytes: int = 0
|
|
1099
|
+
balance_before_bytes: "int | None" = None
|
|
1100
|
+
balance_after_bytes: "int | None" = None
|
|
1101
|
+
mode: str = ""
|
|
1102
|
+
process_write_bytes: "int | None" = None
|
|
1103
|
+
write_status: str = "not_sampled"
|
|
1104
|
+
write_overlap: bool = False
|
|
1105
|
+
skip_reason: "str | None" = None
|
|
1106
|
+
# Q8: the deletion reservation's inputs, so the charge is recomputable
|
|
1107
|
+
# from the record alone (`recompute_charge`).
|
|
1108
|
+
page_count: "int | None" = None
|
|
1109
|
+
usable_size: "int | None" = None
|
|
1110
|
+
page_size: "int | None" = None
|
|
1111
|
+
pointer_map_cap: "int | None" = None
|
|
1112
|
+
reservation_version: "int | None" = None
|
|
1113
|
+
# Q9: a reclaim chunk's plan inputs and outcome (§5.5). The planned page
|
|
1114
|
+
# set itself is receipt-only detail and stays out of the record.
|
|
1115
|
+
planner_version: "int | None" = None
|
|
1116
|
+
sqlite_source_id: "str | None" = None
|
|
1117
|
+
freelist_count: "int | None" = None
|
|
1118
|
+
steps: "int | None" = None
|
|
1119
|
+
kind_free: "int | None" = None
|
|
1120
|
+
kind_overflow: "int | None" = None
|
|
1121
|
+
kind_leaf: "int | None" = None
|
|
1122
|
+
kind_interior: "int | None" = None
|
|
1123
|
+
identified_pages: "int | None" = None
|
|
1124
|
+
unidentified_pages: "int | None" = None
|
|
1125
|
+
fixed_bytes: "int | None" = None
|
|
1126
|
+
plan_digest: "str | None" = None
|
|
1127
|
+
reader_status: "str | None" = None
|
|
1128
|
+
inspected_bytes: "int | None" = None
|
|
1129
|
+
inspection_ms: "int | None" = None
|
|
1130
|
+
freelist_reduction: "int | None" = None
|
|
1131
|
+
page_count_reduction: "int | None" = None
|
|
1132
|
+
|
|
1133
|
+
def as_phase_payload(self) -> dict:
|
|
1134
|
+
return {
|
|
1135
|
+
"op_id": self.op_id,
|
|
1136
|
+
"started_at": _iso(self.started_at),
|
|
1137
|
+
"duration_s": self.duration_s,
|
|
1138
|
+
"rows": self.rows,
|
|
1139
|
+
"pages_reclaimed": self.pages,
|
|
1140
|
+
"charged_bytes": self.charged_bytes,
|
|
1141
|
+
"process_write_bytes": self.process_write_bytes,
|
|
1142
|
+
"write_status": self.write_status,
|
|
1143
|
+
"write_overlap": self.write_overlap,
|
|
1144
|
+
"balance_before_bytes": self.balance_before_bytes,
|
|
1145
|
+
"balance_after_bytes": self.balance_after_bytes,
|
|
1146
|
+
"outcome": self.outcome,
|
|
1147
|
+
"mode": self.mode,
|
|
1148
|
+
"page_count": self.page_count,
|
|
1149
|
+
"usable_size": self.usable_size,
|
|
1150
|
+
"page_size": self.page_size,
|
|
1151
|
+
"pointer_map_cap": self.pointer_map_cap,
|
|
1152
|
+
"reservation_version": self.reservation_version,
|
|
1153
|
+
"skip_reason": self.skip_reason,
|
|
1154
|
+
"planner_version": self.planner_version,
|
|
1155
|
+
"sqlite_source_id": self.sqlite_source_id,
|
|
1156
|
+
"freelist_count": self.freelist_count,
|
|
1157
|
+
"steps": self.steps,
|
|
1158
|
+
"kind_free": self.kind_free,
|
|
1159
|
+
"kind_overflow": self.kind_overflow,
|
|
1160
|
+
"kind_leaf": self.kind_leaf,
|
|
1161
|
+
"kind_interior": self.kind_interior,
|
|
1162
|
+
"identified_pages": self.identified_pages,
|
|
1163
|
+
"unidentified_pages": self.unidentified_pages,
|
|
1164
|
+
"fixed_bytes": self.fixed_bytes,
|
|
1165
|
+
"plan_digest": self.plan_digest,
|
|
1166
|
+
"reader_status": self.reader_status,
|
|
1167
|
+
"inspected_bytes": self.inspected_bytes,
|
|
1168
|
+
"inspection_ms": self.inspection_ms,
|
|
1169
|
+
"freelist_reduction": self.freelist_reduction,
|
|
1170
|
+
"page_count_reduction": self.page_count_reduction,
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _result(kind, outcome, state, started_at, t0, clock, observation, **kw):
|
|
1175
|
+
return OperationResult(
|
|
1176
|
+
op_id=state.op_seq, kind=kind, outcome=outcome, started_at=started_at,
|
|
1177
|
+
duration_s=max(clock() - t0, 0.0),
|
|
1178
|
+
process_write_bytes=observation.process_write_bytes,
|
|
1179
|
+
write_status=observation.status, write_overlap=observation.overlap,
|
|
1180
|
+
**kw)
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
def delete_group(conn: sqlite3.Connection, group: ExpiredGroup, *,
|
|
1184
|
+
now_utc: dt.datetime, cutoff_iso: str,
|
|
1185
|
+
clock: "Callable[[], float]" = time.monotonic,
|
|
1186
|
+
visit_clock0: "float | None" = None) -> OperationResult:
|
|
1187
|
+
"""One whole expired group as one paced operation (spec §5.4 steps 1–4).
|
|
1188
|
+
|
|
1189
|
+
``visit_clock0`` is the visit clock at the visit's start: the charge's
|
|
1190
|
+
ledger bucket is the operation's own start, ``now_utc`` plus the elapsed
|
|
1191
|
+
visit time (PR-13 (a)), while the allowance accrues at ``now_utc``."""
|
|
1192
|
+
started_at = now_utc.astimezone(UTC)
|
|
1193
|
+
rows = group_row_count(conn, group)
|
|
1194
|
+
fallback = rows > DELETION_FALLBACK_ROW_THRESHOLD
|
|
1195
|
+
mode = "fallback" if fallback else "spill_free"
|
|
1196
|
+
# temp_store may only change outside a transaction; the provider flocks
|
|
1197
|
+
# the visit holds keep `rows` exact until the DELETE.
|
|
1198
|
+
conn.execute(f"PRAGMA temp_store = {'FILE' if fallback else 'MEMORY'}")
|
|
1199
|
+
interval = _lib_write_io.begin_interval("maintenance")
|
|
1200
|
+
t0 = clock()
|
|
1201
|
+
op_started = _operation_start(started_at, t0, visit_clock0)
|
|
1202
|
+
try:
|
|
1203
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
1204
|
+
state = PacingState.from_record(read_reclaim_pending(conn), started_at)
|
|
1205
|
+
before = state.available(started_at)
|
|
1206
|
+
if before < 0:
|
|
1207
|
+
conn.rollback()
|
|
1208
|
+
observation = _lib_write_io.end_interval(interval)
|
|
1209
|
+
return _result("delete", "gated", state, started_at, t0, clock,
|
|
1210
|
+
observation, balance_before_bytes=before,
|
|
1211
|
+
balance_after_bytes=before, mode=mode)
|
|
1212
|
+
# Q8: the geometry is read inside this transaction, before the
|
|
1213
|
+
# reservation is recorded, so M_cap bounds this operation's file.
|
|
1214
|
+
geometry = deletion_geometry(conn)
|
|
1215
|
+
cap = pointer_map_cap(geometry.page_count, geometry.usable_size,
|
|
1216
|
+
rows, geometry.page_size)
|
|
1217
|
+
reservation = deletion_reservation(
|
|
1218
|
+
rows, page_count=geometry.page_count,
|
|
1219
|
+
usable_size=geometry.usable_size, page_size=geometry.page_size)
|
|
1220
|
+
charged = dataclasses.replace(
|
|
1221
|
+
state.charge(reservation, started_at, started_utc=op_started),
|
|
1222
|
+
continuation_cutoff=cutoff_iso)
|
|
1223
|
+
_write_reclaim_pending(conn, charged.to_record())
|
|
1224
|
+
_delete_group_rows(conn, group)
|
|
1225
|
+
conn.commit()
|
|
1226
|
+
except BaseException:
|
|
1227
|
+
conn.rollback()
|
|
1228
|
+
_lib_write_io.end_interval(interval)
|
|
1229
|
+
raise
|
|
1230
|
+
observation = _lib_write_io.end_interval(interval)
|
|
1231
|
+
_lib_write_io.mark_deletion(observation, rows=rows)
|
|
1232
|
+
return _result("delete", "ok", charged, started_at, t0, clock, observation,
|
|
1233
|
+
rows=rows, charged_bytes=reservation,
|
|
1234
|
+
balance_before_bytes=before,
|
|
1235
|
+
balance_after_bytes=charged.balance_bytes, mode=mode,
|
|
1236
|
+
page_count=geometry.page_count,
|
|
1237
|
+
usable_size=geometry.usable_size,
|
|
1238
|
+
page_size=geometry.page_size, pointer_map_cap=cap,
|
|
1239
|
+
reservation_version=RESERVATION_VERSION)
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
def _run_incremental_vacuum_chunk(conn: sqlite3.Connection, steps: int) -> None:
|
|
1243
|
+
"""Run ``steps`` ``incremental_vacuum(1)`` statements INSIDE the caller's
|
|
1244
|
+
transaction, with no other SQL between them.
|
|
1245
|
+
|
|
1246
|
+
One step per statement: a statement limited to one page is complete
|
|
1247
|
+
after its first step on every Python, so CPython 3.11's single-step
|
|
1248
|
+
zero-column cursor cannot shorten it, and no `executescript` is needed -
|
|
1249
|
+
that call would COMMIT the open transaction first and split the
|
|
1250
|
+
reservation from its mutation."""
|
|
1251
|
+
for _ in range(int(steps)):
|
|
1252
|
+
conn.execute("PRAGMA incremental_vacuum(1)").fetchall()
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def _sqlite_version_info() -> tuple:
|
|
1256
|
+
"""The runtime SQLite's version (a seam G3q injects)."""
|
|
1257
|
+
return tuple(sqlite3.sqlite_version_info)
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
class _ChunkRefused(Exception):
|
|
1261
|
+
def __init__(self, reason: str, *, inspected_bytes=None,
|
|
1262
|
+
inspection_ms=None):
|
|
1263
|
+
super().__init__(reason)
|
|
1264
|
+
self.reason = reason
|
|
1265
|
+
self.inspected_bytes = inspected_bytes
|
|
1266
|
+
self.inspection_ms = inspection_ms
|
|
1267
|
+
|
|
1268
|
+
|
|
1269
|
+
def _request_plan(db_path, *, page_size: int, page_count: int,
|
|
1270
|
+
freelist_count: int, max_steps: int,
|
|
1271
|
+
deadline_s: float) -> "_planner.ReclaimPlan":
|
|
1272
|
+
"""Plan the chunk in a fresh helper process while this process holds
|
|
1273
|
+
the chunk's `BEGIN IMMEDIATE` (spec §5.4 "Read path"). Raises
|
|
1274
|
+
`_ChunkRefused` with a typed reason; never opens the store itself."""
|
|
1275
|
+
deadline_s = max(_PLANNER_MIN_DEADLINE_SECONDS, float(deadline_s))
|
|
1276
|
+
request = {"db": str(db_path), "pageSize": int(page_size),
|
|
1277
|
+
"pageCount": int(page_count),
|
|
1278
|
+
"freelistCount": int(freelist_count),
|
|
1279
|
+
"maxSteps": int(max_steps),
|
|
1280
|
+
"budgetBytes": RECLAIM_CHUNK_BUDGET_BYTES,
|
|
1281
|
+
"fixedBytes": RECLAIM_FIXED_BYTES,
|
|
1282
|
+
"readBudgetBytes": _PLANNER_READ_BUDGET_BYTES,
|
|
1283
|
+
"deadlineMs": int(deadline_s * 1000)}
|
|
1284
|
+
try:
|
|
1285
|
+
child = subprocess.run(
|
|
1286
|
+
list(_PLANNER_COMMAND) + ["--plan"], input=json.dumps(request),
|
|
1287
|
+
capture_output=True, text=True, close_fds=True,
|
|
1288
|
+
timeout=deadline_s + _PLANNER_STARTUP_SLACK_SECONDS)
|
|
1289
|
+
except subprocess.TimeoutExpired as exc:
|
|
1290
|
+
raise _ChunkRefused("helper_timeout") from exc
|
|
1291
|
+
except OSError as exc:
|
|
1292
|
+
raise _ChunkRefused("helper_failed") from exc
|
|
1293
|
+
if child.returncode != 0:
|
|
1294
|
+
raise _ChunkRefused("helper_failed")
|
|
1295
|
+
try:
|
|
1296
|
+
answer = json.loads(child.stdout)
|
|
1297
|
+
except ValueError as exc:
|
|
1298
|
+
raise _ChunkRefused("helper_failed") from exc
|
|
1299
|
+
if not isinstance(answer, dict):
|
|
1300
|
+
raise _ChunkRefused("helper_failed")
|
|
1301
|
+
if answer.get("ok") is not True:
|
|
1302
|
+
reason = answer.get("reason")
|
|
1303
|
+
raise _ChunkRefused(
|
|
1304
|
+
reason if reason in _planner.REFUSAL_REASONS else "helper_failed",
|
|
1305
|
+
inspected_bytes=_record_int(answer, "inspectedBytes"),
|
|
1306
|
+
inspection_ms=_record_int(answer, "inspectionMs"))
|
|
1307
|
+
try:
|
|
1308
|
+
plan = _planner.ReclaimPlan.from_wire(answer.get("plan"))
|
|
1309
|
+
except (ValueError, TypeError, KeyError) as exc:
|
|
1310
|
+
raise _ChunkRefused("helper_failed") from exc
|
|
1311
|
+
if ((plan.page_size, plan.page_count, plan.freelist_count)
|
|
1312
|
+
!= (int(page_size), int(page_count), int(freelist_count))
|
|
1313
|
+
or plan.steps > int(max_steps)
|
|
1314
|
+
or plan.fixed_bytes != RECLAIM_FIXED_BYTES):
|
|
1315
|
+
raise _ChunkRefused("wal_endpoint_mismatch")
|
|
1316
|
+
return plan
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def _plan_fields(plan: "_planner.ReclaimPlan") -> dict:
|
|
1320
|
+
return dict(
|
|
1321
|
+
planner_version=_planner.PLANNER_VERSION,
|
|
1322
|
+
page_size=plan.page_size, usable_size=plan.usable_size,
|
|
1323
|
+
page_count=plan.page_count, freelist_count=plan.freelist_count,
|
|
1324
|
+
steps=plan.steps, kind_free=plan.kinds["free"],
|
|
1325
|
+
kind_overflow=plan.kinds["overflow"], kind_leaf=plan.kinds["leaf"],
|
|
1326
|
+
kind_interior=plan.kinds["interior"],
|
|
1327
|
+
identified_pages=len(plan.identified),
|
|
1328
|
+
unidentified_pages=plan.unidentified, fixed_bytes=plan.fixed_bytes,
|
|
1329
|
+
plan_digest=plan.digest, reader_status=plan.reader_status,
|
|
1330
|
+
inspected_bytes=plan.inspected_bytes,
|
|
1331
|
+
inspection_ms=plan.inspection_ms,
|
|
1332
|
+
reservation_version=RESERVATION_VERSION)
|
|
1333
|
+
|
|
1334
|
+
|
|
1335
|
+
def reclaim_chunk(conn: sqlite3.Connection, *, now_utc: dt.datetime,
|
|
1336
|
+
pages: int = RECLAIM_MAX_CHUNK_PAGES,
|
|
1337
|
+
next_attempt_at: "dt.datetime | None" = None,
|
|
1338
|
+
clock: "Callable[[], float]" = time.monotonic,
|
|
1339
|
+
deadline_s: "float | None" = None,
|
|
1340
|
+
visit_clock0: "float | None" = None) -> OperationResult:
|
|
1341
|
+
"""One planned reclaim chunk of at most ``pages`` (<= 16) steps as one
|
|
1342
|
+
paced operation (spec §5.4, Q9), on a connection from
|
|
1343
|
+
`open_reclaim_connection` (it never checkpoints).
|
|
1344
|
+
|
|
1345
|
+
In ONE `BEGIN IMMEDIATE` transaction: the record precondition, then the
|
|
1346
|
+
plan (a helper subprocess reads the snapshot this transaction holds),
|
|
1347
|
+
then the planned `incremental_vacuum(1)` steps with no other SQL between
|
|
1348
|
+
the plan's reads and the first step, then the frozen reservation's
|
|
1349
|
+
state write (a same-key `UPDATE`), then `COMMIT`. Any refusal rolls back
|
|
1350
|
+
with no vacuum and no charge and returns `outcome="skipped"` with its
|
|
1351
|
+
typed `skip_reason`."""
|
|
1352
|
+
started_at = now_utc.astimezone(UTC)
|
|
1353
|
+
max_steps = max(1, min(int(pages), RECLAIM_MAX_CHUNK_PAGES))
|
|
1354
|
+
deadline = (_PLANNER_DEADLINE_SECONDS if deadline_s is None
|
|
1355
|
+
else float(deadline_s))
|
|
1356
|
+
interval = _lib_write_io.begin_interval("maintenance")
|
|
1357
|
+
t0 = clock()
|
|
1358
|
+
op_started = _operation_start(started_at, t0, visit_clock0)
|
|
1359
|
+
source_id = None
|
|
1360
|
+
state = PacingState(balance_bytes=0, as_of=started_at)
|
|
1361
|
+
before = None
|
|
1362
|
+
|
|
1363
|
+
def skipped(reason, **fields):
|
|
1364
|
+
conn.rollback()
|
|
1365
|
+
observation = _lib_write_io.end_interval(interval)
|
|
1366
|
+
return _result("reclaim", "skipped", state, started_at, t0, clock,
|
|
1367
|
+
observation, balance_before_bytes=before,
|
|
1368
|
+
balance_after_bytes=before, mode="reclaim",
|
|
1369
|
+
skip_reason=reason, reader_status=fields.pop(
|
|
1370
|
+
"reader_status", reason),
|
|
1371
|
+
sqlite_source_id=source_id, **fields)
|
|
313
1372
|
|
|
1373
|
+
try:
|
|
1374
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
1375
|
+
if reclaim_record_length(conn) != RECLAIM_RECORD_LENGTH:
|
|
1376
|
+
# The in-place state write needs the L-byte row; the attempt
|
|
1377
|
+
# normalizes it in its own transaction before any chunk.
|
|
1378
|
+
return skipped("state_record_unnormalized", reader_status=None)
|
|
1379
|
+
state = PacingState.from_record(read_reclaim_pending(conn), started_at)
|
|
1380
|
+
before = state.available(started_at)
|
|
1381
|
+
geometry = store_geometry(conn)
|
|
1382
|
+
free = 0 if geometry is None else geometry.freelist_count
|
|
1383
|
+
if before < 0 or free <= 0:
|
|
1384
|
+
conn.rollback()
|
|
1385
|
+
observation = _lib_write_io.end_interval(interval)
|
|
1386
|
+
return _result("reclaim", "gated" if before < 0 else "no_progress",
|
|
1387
|
+
state, started_at, t0, clock, observation,
|
|
1388
|
+
balance_before_bytes=before,
|
|
1389
|
+
balance_after_bytes=before, mode="reclaim")
|
|
1390
|
+
source_id = str(conn.execute(
|
|
1391
|
+
"SELECT sqlite_source_id()").fetchone()[0])[:96]
|
|
1392
|
+
if not _planner.sqlite_audited(_sqlite_version_info()):
|
|
1393
|
+
return skipped("sqlite_unaudited")
|
|
1394
|
+
if int(conn.execute("PRAGMA auto_vacuum").fetchone()[0]) != 2:
|
|
1395
|
+
return skipped("unsupported_geometry")
|
|
1396
|
+
reserved = reserved_bytes(conn)
|
|
1397
|
+
usable = geometry.page_size - (
|
|
1398
|
+
_MAX_RESERVED_BYTES if reserved is None else reserved)
|
|
1399
|
+
if reserved != 0 or not record_fits_locally(usable):
|
|
1400
|
+
return skipped("unsupported_geometry")
|
|
1401
|
+
db_path = _resolve_main_db_path(conn)
|
|
1402
|
+
try:
|
|
1403
|
+
plan = _request_plan(
|
|
1404
|
+
db_path, page_size=geometry.page_size,
|
|
1405
|
+
page_count=geometry.page_count, freelist_count=free,
|
|
1406
|
+
max_steps=max_steps, deadline_s=deadline)
|
|
1407
|
+
except _ChunkRefused as refused:
|
|
1408
|
+
return skipped(refused.reason,
|
|
1409
|
+
inspected_bytes=refused.inspected_bytes,
|
|
1410
|
+
inspection_ms=refused.inspection_ms)
|
|
1411
|
+
# No other SQL between the plan's reads and the first step.
|
|
1412
|
+
_run_incremental_vacuum_chunk(conn, plan.steps)
|
|
1413
|
+
after = store_geometry(conn)
|
|
1414
|
+
reservation = plan.reservation_bytes
|
|
1415
|
+
remaining = after.freelist_count
|
|
1416
|
+
charged = dataclasses.replace(
|
|
1417
|
+
state.charge(reservation, started_at, started_utc=op_started),
|
|
1418
|
+
eligible=True,
|
|
1419
|
+
next_attempt_at=next_attempt_at or state.next_attempt_at,
|
|
1420
|
+
last_failure=None,
|
|
1421
|
+
progress={"freelist_pages": remaining,
|
|
1422
|
+
"page_size": after.page_size,
|
|
1423
|
+
"observed_at": _iso(started_at)},
|
|
1424
|
+
legacy={**state.legacy,
|
|
1425
|
+
"freelist_count": remaining,
|
|
1426
|
+
"unreclaimed_bytes": remaining * after.page_size,
|
|
1427
|
+
"attempted_at": _iso(started_at),
|
|
1428
|
+
"made_progress": True, "deadline_hit": False},
|
|
1429
|
+
)
|
|
1430
|
+
try:
|
|
1431
|
+
_write_reclaim_state_in_chunk(conn, charged.to_record())
|
|
1432
|
+
except StateRecordOversize:
|
|
1433
|
+
return skipped("state_record_oversize", **{
|
|
1434
|
+
k: v for k, v in _plan_fields(plan).items()
|
|
1435
|
+
if k not in ("reader_status",)})
|
|
1436
|
+
conn.commit()
|
|
1437
|
+
except BaseException:
|
|
1438
|
+
conn.rollback()
|
|
1439
|
+
_lib_write_io.end_interval(interval)
|
|
1440
|
+
raise
|
|
1441
|
+
observation = _lib_write_io.end_interval(interval)
|
|
1442
|
+
freelist_reduction = free - remaining
|
|
1443
|
+
return _result("reclaim", "ok", charged, started_at, t0, clock,
|
|
1444
|
+
observation, pages=freelist_reduction,
|
|
1445
|
+
charged_bytes=reservation, balance_before_bytes=before,
|
|
1446
|
+
balance_after_bytes=charged.balance_bytes, mode="reclaim",
|
|
1447
|
+
sqlite_source_id=source_id,
|
|
1448
|
+
freelist_reduction=freelist_reduction,
|
|
1449
|
+
page_count_reduction=geometry.page_count - after.page_count,
|
|
1450
|
+
**_plan_fields(plan))
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
# ── the visit ─────────────────────────────────────────────────────────────
|
|
314
1454
|
|
|
315
1455
|
def _retention_due(conn: sqlite3.Connection, now_utc: dt.datetime) -> bool:
|
|
316
1456
|
"""True iff the prune has never run or last ran more than 24h ago."""
|
|
@@ -340,105 +1480,9 @@ def _stamp_retention_prune(conn: sqlite3.Connection, now_utc: dt.datetime) -> No
|
|
|
340
1480
|
)
|
|
341
1481
|
|
|
342
1482
|
|
|
343
|
-
def _run_incremental_vacuum_chunk(conn: sqlite3.Connection, pages: int) -> int:
|
|
344
|
-
"""Reclaim at most ``pages`` freelist pages; return the freelist count after.
|
|
345
|
-
|
|
346
|
-
The one seam every chunk goes through, so a test can replace it to hold the
|
|
347
|
-
real SQLite write lock at a barrier and observe what a concurrent reader
|
|
348
|
-
does. ``executescript()`` routes through ``sqlite3_exec()``, which steps
|
|
349
|
-
zero-column pragma rows through ``SQLITE_DONE`` on Python/SQLite
|
|
350
|
-
combinations where ``Cursor.fetchall()`` can stop after the first row
|
|
351
|
-
(public Linux 3.11).
|
|
352
|
-
"""
|
|
353
|
-
conn.executescript(f"PRAGMA incremental_vacuum({int(pages)});")
|
|
354
|
-
return int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
def _reclaim_incremental_vacuum(
|
|
358
|
-
conn: sqlite3.Connection,
|
|
359
|
-
*,
|
|
360
|
-
deadline_seconds: float = RECLAIM_DEADLINE_SECONDS,
|
|
361
|
-
clock: "Callable[[], float]" = time.monotonic,
|
|
362
|
-
) -> ReclaimOutcome:
|
|
363
|
-
"""Reclaim freelist pages under a monotonic wall-clock budget (#780).
|
|
364
|
-
|
|
365
|
-
The pass runs under the maintenance flock and both provider flocks, so its
|
|
366
|
-
cost is a cost every reader and both ingesters pay. It therefore stops at
|
|
367
|
-
the deadline and leaves the remainder for the next pass rather than driving
|
|
368
|
-
the freelist to zero however long that takes.
|
|
369
|
-
|
|
370
|
-
Chunks start small and adapt to measured throughput, so a slow disk cannot
|
|
371
|
-
overshoot the budget on the first chunk and a fast one is not held to 64
|
|
372
|
-
pages a chunk for the whole pass. The freelist is re-read after every
|
|
373
|
-
chunk, which is also how a chunk that reclaimed nothing is detected.
|
|
374
|
-
"""
|
|
375
|
-
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
|
|
376
|
-
before = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
|
377
|
-
started = clock()
|
|
378
|
-
if before <= 0:
|
|
379
|
-
return ReclaimOutcome(
|
|
380
|
-
freelist_before=before, freelist_after=before, page_size=page_size)
|
|
381
|
-
remaining = before
|
|
382
|
-
pages = RECLAIM_INITIAL_PAGES
|
|
383
|
-
# The SLOWEST rate observed in THIS pass, not the most recent one. A cold
|
|
384
|
-
# store's per-page cost is about twenty times its warm cost, so sizing the
|
|
385
|
-
# next chunk from the chunk just measured is optimistic exactly when it
|
|
386
|
-
# matters: measured on a clone of the production store, that estimator ran
|
|
387
|
-
# a first pass to 7.9 s against a 2.0 s budget. Taking the minimum makes
|
|
388
|
-
# one slow chunk hold the rest of the pass conservative, and the estimate
|
|
389
|
-
# resets each pass, so a warm pass still ramps.
|
|
390
|
-
slowest_rate = None
|
|
391
|
-
deadline_hit = False
|
|
392
|
-
while remaining > 0:
|
|
393
|
-
elapsed = clock() - started
|
|
394
|
-
if elapsed >= deadline_seconds:
|
|
395
|
-
deadline_hit = True
|
|
396
|
-
break
|
|
397
|
-
requested = min(remaining, pages)
|
|
398
|
-
chunk_started = clock()
|
|
399
|
-
after = _run_incremental_vacuum_chunk(conn, requested)
|
|
400
|
-
chunk_elapsed = max(clock() - chunk_started, 0.0)
|
|
401
|
-
if after >= remaining:
|
|
402
|
-
# The chunk reclaimed nothing. Continuing would spin against a
|
|
403
|
-
# store that is not releasing pages.
|
|
404
|
-
remaining = after
|
|
405
|
-
break
|
|
406
|
-
if chunk_elapsed > 0:
|
|
407
|
-
# Aim each chunk at a quarter of the remaining budget, so the pass
|
|
408
|
-
# takes several measured steps rather than one guess — and never
|
|
409
|
-
# more than DOUBLE the chunk just measured, so a cold first chunk
|
|
410
|
-
# cannot extrapolate the pass past its deadline.
|
|
411
|
-
rate = (remaining - after) / chunk_elapsed
|
|
412
|
-
slowest_rate = (
|
|
413
|
-
rate if slowest_rate is None else min(slowest_rate, rate))
|
|
414
|
-
budget_left = max(deadline_seconds - (clock() - started), 0.0)
|
|
415
|
-
pages = int(max(
|
|
416
|
-
RECLAIM_INITIAL_PAGES,
|
|
417
|
-
min(RECLAIM_MAX_PAGES,
|
|
418
|
-
slowest_rate * budget_left / 4,
|
|
419
|
-
requested * RECLAIM_CHUNK_GROWTH_FACTOR),
|
|
420
|
-
))
|
|
421
|
-
remaining = after
|
|
422
|
-
return ReclaimOutcome(
|
|
423
|
-
freelist_before=before,
|
|
424
|
-
freelist_after=remaining,
|
|
425
|
-
page_size=page_size,
|
|
426
|
-
pages_reclaimed=max(0, before - remaining),
|
|
427
|
-
duration_s=max(clock() - started, 0.0),
|
|
428
|
-
deadline_hit=deadline_hit,
|
|
429
|
-
)
|
|
430
|
-
|
|
431
|
-
|
|
432
1483
|
def _resolve_main_db_path(conn: sqlite3.Connection):
|
|
433
|
-
"""The file behind this connection's ``main`` schema
|
|
434
|
-
|
|
435
|
-
Read off the connection rather than assumed, because the orchestrator is
|
|
436
|
-
called with a conversations connection by the dashboard and with a cache
|
|
437
|
-
connection by the from-zero-replay callers; measuring the wrong family's
|
|
438
|
-
size would report a reclaim against a file the pass never touched. Falls
|
|
439
|
-
back to the conversations path when the connection reports no file, which
|
|
440
|
-
is the in-memory case.
|
|
441
|
-
"""
|
|
1484
|
+
"""The file behind this connection's ``main`` schema; the conversations
|
|
1485
|
+
path for an in-memory connection."""
|
|
442
1486
|
try:
|
|
443
1487
|
for _seq, name, file_name in conn.execute("PRAGMA database_list"):
|
|
444
1488
|
if name == "main" and file_name:
|
|
@@ -448,149 +1492,202 @@ def _resolve_main_db_path(conn: sqlite3.Connection):
|
|
|
448
1492
|
return _cctally_core.CONVERSATIONS_DB_PATH
|
|
449
1493
|
|
|
450
1494
|
|
|
451
|
-
def
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
base = str(path)
|
|
455
|
-
for member in (base, base + "-wal", base + "-shm"):
|
|
456
|
-
try:
|
|
457
|
-
total += os.path.getsize(member)
|
|
458
|
-
except OSError:
|
|
459
|
-
continue
|
|
460
|
-
return total
|
|
1495
|
+
def _report(record_phase, result: OperationResult) -> None:
|
|
1496
|
+
if record_phase is not None:
|
|
1497
|
+
record_phase(result.kind, result.as_phase_payload())
|
|
461
1498
|
|
|
462
1499
|
|
|
463
|
-
def
|
|
464
|
-
"""
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
WAL until it is truncated, so clearing the pending state at
|
|
469
|
-
`freelist_count == 0` would declare success while the space is still
|
|
470
|
-
occupied. Returns None when the checkpoint could not run at all.
|
|
471
|
-
"""
|
|
1500
|
+
def _update_record(db_path, mutate) -> None:
|
|
1501
|
+
"""A state-only record write (no mutation, no charge) on a connection
|
|
1502
|
+
that never checkpoints; `mutate(record) -> record | None` (None = no
|
|
1503
|
+
change, no write)."""
|
|
1504
|
+
conn = _open_state_connection(db_path)
|
|
472
1505
|
try:
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
1506
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
1507
|
+
try:
|
|
1508
|
+
record = read_reclaim_pending(conn)
|
|
1509
|
+
updated = mutate(record)
|
|
1510
|
+
if updated is None:
|
|
1511
|
+
conn.rollback()
|
|
1512
|
+
return
|
|
1513
|
+
_write_reclaim_pending(conn, updated)
|
|
1514
|
+
conn.commit()
|
|
1515
|
+
except BaseException:
|
|
1516
|
+
conn.rollback()
|
|
1517
|
+
raise
|
|
1518
|
+
finally:
|
|
1519
|
+
conn.close()
|
|
1520
|
+
|
|
1521
|
+
|
|
1522
|
+
def _end_reclaim_episode(db_path) -> None:
|
|
1523
|
+
def mutate(record):
|
|
1524
|
+
if not isinstance(record, dict) or record.get("eligible") is not True:
|
|
1525
|
+
return None
|
|
1526
|
+
return {**record, "eligible": False}
|
|
1527
|
+
_update_record(db_path, mutate)
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def _record_reclaim_failure(db_path, reason: str, now_utc: dt.datetime) -> None:
|
|
1531
|
+
"""Durable only when the reason changes, so repeated skips rewrite
|
|
1532
|
+
nothing."""
|
|
1533
|
+
def mutate(record):
|
|
1534
|
+
record = dict(record) if isinstance(record, dict) else {}
|
|
1535
|
+
prior = record.get("last_failure")
|
|
1536
|
+
if isinstance(prior, dict) and prior.get("reason") == reason:
|
|
1537
|
+
return None
|
|
1538
|
+
if record.get("policy_version") is None:
|
|
1539
|
+
record = PacingState.from_record(record, now_utc).to_record()
|
|
1540
|
+
record["last_failure"] = {"reason": reason, "at": _iso(now_utc)}
|
|
1541
|
+
return record
|
|
1542
|
+
_update_record(db_path, mutate)
|
|
1543
|
+
|
|
1544
|
+
|
|
1545
|
+
def _stamp_retention_complete(conn: sqlite3.Connection, now_utc: dt.datetime) -> None:
|
|
1546
|
+
"""Both providers finished: the daily stamp, and the continuation
|
|
1547
|
+
cleared, in one small transaction on the deletion connection."""
|
|
1548
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
483
1549
|
try:
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
1550
|
+
_stamp_retention_prune(conn, now_utc)
|
|
1551
|
+
record = read_reclaim_pending(conn)
|
|
1552
|
+
if isinstance(record, dict) and record.get("continuation_cutoff") is not None:
|
|
1553
|
+
_write_reclaim_pending(conn, {**record, "continuation_cutoff": None})
|
|
1554
|
+
conn.commit()
|
|
1555
|
+
except BaseException:
|
|
1556
|
+
conn.rollback()
|
|
1557
|
+
raise
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
def _run_deletion_visit(db_path, *, now, retention_days, state, record_phase,
|
|
1561
|
+
clock) -> PruneStats:
|
|
1562
|
+
visit_clock0 = clock()
|
|
1563
|
+
cutoff_iso = state.continuation_cutoff or _cutoff_iso(
|
|
1564
|
+
now - dt.timedelta(days=int(retention_days)))
|
|
1565
|
+
counts = {"claude_sessions": 0, "claude_messages": 0,
|
|
1566
|
+
"codex_conversations": 0, "codex_events": 0}
|
|
1567
|
+
complete = True
|
|
1568
|
+
conn = _open_deletion_connection(db_path)
|
|
492
1569
|
try:
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
1570
|
+
groups = expired_groups(conn, cutoff_iso)
|
|
1571
|
+
for index, group in enumerate(groups):
|
|
1572
|
+
result = delete_group(conn, group, now_utc=now,
|
|
1573
|
+
cutoff_iso=cutoff_iso, clock=clock,
|
|
1574
|
+
visit_clock0=visit_clock0)
|
|
1575
|
+
_report(record_phase, result)
|
|
1576
|
+
if result.outcome != "ok":
|
|
1577
|
+
complete = False
|
|
1578
|
+
break
|
|
1579
|
+
if group.provider == "claude":
|
|
1580
|
+
counts["claude_sessions"] += 1
|
|
1581
|
+
counts["claude_messages"] += result.rows
|
|
1582
|
+
else:
|
|
1583
|
+
counts["codex_conversations"] += 1
|
|
1584
|
+
counts["codex_events"] += result.rows
|
|
1585
|
+
if (result.balance_after_bytes is not None
|
|
1586
|
+
and result.balance_after_bytes < 0
|
|
1587
|
+
and index + 1 < len(groups)):
|
|
1588
|
+
complete = False
|
|
1589
|
+
break
|
|
1590
|
+
if complete:
|
|
1591
|
+
_stamp_retention_complete(conn, now)
|
|
1592
|
+
finally:
|
|
1593
|
+
conn.close()
|
|
1594
|
+
return PruneStats(**counts, complete=complete)
|
|
497
1595
|
|
|
498
1596
|
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
1597
|
+
#: A refused attempt writes nothing durable, so its next attempt time is
|
|
1598
|
+
#: kept here, per store, to hold the one-minute cadence (spec §5.4). A durable
|
|
1599
|
+
#: failure (the reclaim connection unavailable, a SQLite error) records its
|
|
1600
|
+
#: reason only when it changes and writes no `next_attempt_at`, so it holds
|
|
1601
|
+
#: the same cadence here too (#901 Amendment 19 PR-4).
|
|
1602
|
+
_REFUSED_UNTIL: "dict[str, dt.datetime]" = {}
|
|
1603
|
+
|
|
1604
|
+
#: The reasons that come from the planner and its helper (in-memory only);
|
|
1605
|
+
#: the connection-settings and SQLite-error skips stay durable, as before.
|
|
1606
|
+
_IN_MEMORY_SKIP_REASONS = frozenset(_planner.REFUSAL_REASONS) | {
|
|
1607
|
+
"helper_failed", "helper_timeout", "state_record_unnormalized",
|
|
1608
|
+
"state_record_oversize"}
|
|
1609
|
+
|
|
508
1610
|
|
|
1611
|
+
def _refusal_due(db_path, now: dt.datetime) -> bool:
|
|
1612
|
+
until = _REFUSED_UNTIL.get(str(db_path))
|
|
1613
|
+
return until is None or now >= until
|
|
509
1614
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
1615
|
+
|
|
1616
|
+
def _publish_refusal(db_path, reason: "str | None", now: dt.datetime) -> None:
|
|
1617
|
+
"""The in-memory pass record of a refused attempt (no durable write):
|
|
1618
|
+
`writeIo.reclaimRefusal` for the doctor, and the next attempt time."""
|
|
514
1619
|
try:
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
1620
|
+
if reason is None:
|
|
1621
|
+
_REFUSED_UNTIL.pop(str(db_path), None)
|
|
1622
|
+
_lib_write_io.clear_reclaim_refusal()
|
|
1623
|
+
else:
|
|
1624
|
+
_REFUSED_UNTIL[str(db_path)] = now + dt.timedelta(
|
|
1625
|
+
seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
|
|
1626
|
+
_lib_write_io.set_reclaim_refusal(reason, now)
|
|
1627
|
+
except Exception: # noqa: BLE001 — telemetry must not fail maintenance
|
|
1628
|
+
pass
|
|
518
1629
|
|
|
519
1630
|
|
|
520
|
-
def
|
|
521
|
-
"""
|
|
522
|
-
|
|
1631
|
+
def _hold_after_failure(db_path, now: dt.datetime) -> None:
|
|
1632
|
+
"""A durable reclaim failure waits one interval before the next attempt
|
|
1633
|
+
(spec §5.4: "attempted at most once per minute"). In memory, like a
|
|
1634
|
+
refusal's hold, so a repeated failure still rewrites nothing; it publishes
|
|
1635
|
+
no refusal, because the failure is already the record's `last_failure`.
|
|
1636
|
+
The next chunk that runs clears it (`_publish_refusal(..., None, ...)`)."""
|
|
1637
|
+
_REFUSED_UNTIL[str(db_path)] = now + dt.timedelta(
|
|
1638
|
+
seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
|
|
523
1639
|
|
|
524
1640
|
|
|
525
|
-
def
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
1641
|
+
def _run_reclaim_attempt(db_path, *, now, record_phase, clock) -> None:
|
|
1642
|
+
next_attempt = now + dt.timedelta(seconds=RECLAIM_ATTEMPT_INTERVAL_SECONDS)
|
|
1643
|
+
try:
|
|
1644
|
+
conn = _open_reclaim_connection(db_path)
|
|
1645
|
+
except ReclaimUnavailable as exc:
|
|
1646
|
+
_hold_after_failure(db_path, now)
|
|
1647
|
+
_record_reclaim_failure(db_path, exc.reason, now)
|
|
1648
|
+
_report(record_phase, OperationResult(
|
|
1649
|
+
op_id=0, kind="reclaim", outcome="skipped", started_at=now,
|
|
1650
|
+
duration_s=0.0, mode="reclaim", skip_reason=exc.reason,
|
|
1651
|
+
reader_status=exc.reason))
|
|
1652
|
+
return
|
|
1653
|
+
started = clock()
|
|
1654
|
+
try:
|
|
1655
|
+
# §5.4 "State write": a legacy-length or absent record is normalized
|
|
1656
|
+
# in its own uncharged transaction before the first chunk.
|
|
1657
|
+
normalize_reclaim_record(conn, now)
|
|
1658
|
+
while clock() - started < RECLAIM_DEADLINE_SECONDS:
|
|
1659
|
+
if not reclaim_episode_active(store_geometry(conn), True):
|
|
1660
|
+
conn.close()
|
|
1661
|
+
conn = None
|
|
1662
|
+
_end_reclaim_episode(db_path)
|
|
1663
|
+
break
|
|
1664
|
+
remaining = RECLAIM_DEADLINE_SECONDS - (clock() - started)
|
|
1665
|
+
result = reclaim_chunk(conn, now_utc=now, next_attempt_at=next_attempt,
|
|
1666
|
+
clock=clock, deadline_s=remaining,
|
|
1667
|
+
visit_clock0=started)
|
|
1668
|
+
_report(record_phase, result)
|
|
1669
|
+
if result.outcome == "skipped" and (
|
|
1670
|
+
result.skip_reason in _IN_MEMORY_SKIP_REASONS):
|
|
1671
|
+
_publish_refusal(db_path, result.skip_reason, now)
|
|
1672
|
+
break
|
|
1673
|
+
if result.outcome == "ok":
|
|
1674
|
+
_publish_refusal(db_path, None, now)
|
|
1675
|
+
if result.outcome != "ok" or (result.balance_after_bytes or 0) < 0:
|
|
1676
|
+
break
|
|
1677
|
+
except sqlite3.Error:
|
|
1678
|
+
_hold_after_failure(db_path, now)
|
|
1679
|
+
_record_reclaim_failure(db_path, "sqlite_error", now)
|
|
1680
|
+
finally:
|
|
1681
|
+
if conn is not None:
|
|
1682
|
+
conn.close()
|
|
530
1683
|
|
|
531
1684
|
|
|
532
|
-
def
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
clock: "Callable[[], float]" = time.monotonic,
|
|
539
|
-
) -> "dict":
|
|
540
|
-
"""One bounded reclaim + checkpoint pass, with durable pending state.
|
|
541
|
-
|
|
542
|
-
Returns a phase record: ``{reclaim: {...}, checkpoint: {...}, pending: ...}``.
|
|
543
|
-
The pending record is retained until a MEASURED reduction in the database
|
|
544
|
-
family's size, or the checkpoint's own result, shows the space was really
|
|
545
|
-
returned. A pass that made no progress stays pending and is reported; it is
|
|
546
|
-
never declared complete.
|
|
547
|
-
"""
|
|
548
|
-
path = _resolve_main_db_path(conn) if db_path is None else db_path
|
|
549
|
-
family_before = _db_family_bytes(path)
|
|
550
|
-
outcome = _reclaim_incremental_vacuum(
|
|
551
|
-
conn, deadline_seconds=deadline_seconds, clock=clock)
|
|
552
|
-
checkpoint = _checkpoint_truncate(conn)
|
|
553
|
-
family_after = _db_family_bytes(path)
|
|
554
|
-
returned = max(0, family_before - family_after)
|
|
555
|
-
complete = (
|
|
556
|
-
outcome.freelist_drained
|
|
557
|
-
and checkpoint is not None
|
|
558
|
-
and checkpoint[0] == 0
|
|
559
|
-
and checkpoint[1] == 0
|
|
560
|
-
)
|
|
561
|
-
state = None
|
|
562
|
-
if not complete:
|
|
563
|
-
state = {
|
|
564
|
-
"freelist_count": outcome.freelist_after,
|
|
565
|
-
"unreclaimed_bytes": outcome.unreclaimed_bytes,
|
|
566
|
-
"attempted_at": now_utc.astimezone(UTC).isoformat(),
|
|
567
|
-
"made_progress": bool(outcome.made_progress),
|
|
568
|
-
"deadline_hit": bool(outcome.deadline_hit),
|
|
569
|
-
}
|
|
570
|
-
# Write and commit ONLY when the record actually changed. A pass over a
|
|
571
|
-
# store with no backlog would otherwise issue a no-op DELETE and a commit
|
|
572
|
-
# on every cycle, which is both pointless and visible to callers counting
|
|
573
|
-
# transaction boundaries.
|
|
574
|
-
if state != read_reclaim_pending(conn):
|
|
575
|
-
_write_reclaim_pending(conn, state)
|
|
576
|
-
conn.commit()
|
|
577
|
-
return {
|
|
578
|
-
"reclaim": {
|
|
579
|
-
"pages_reclaimed": outcome.pages_reclaimed,
|
|
580
|
-
"freelist_before": outcome.freelist_before,
|
|
581
|
-
"freelist_after": outcome.freelist_after,
|
|
582
|
-
"duration_s": outcome.duration_s,
|
|
583
|
-
"deadline_hit": outcome.deadline_hit,
|
|
584
|
-
"made_progress": outcome.made_progress,
|
|
585
|
-
},
|
|
586
|
-
"checkpoint": {
|
|
587
|
-
"result": checkpoint,
|
|
588
|
-
"bytes_returned": returned,
|
|
589
|
-
"family_bytes": family_after,
|
|
590
|
-
},
|
|
591
|
-
"pending": state,
|
|
592
|
-
"complete": complete,
|
|
593
|
-
}
|
|
1685
|
+
def _publish_record(record) -> None:
|
|
1686
|
+
"""Hand the latest ledger to this process's `writeIo.maintenance`."""
|
|
1687
|
+
try:
|
|
1688
|
+
_lib_write_io.set_maintenance_record(record)
|
|
1689
|
+
except Exception: # noqa: BLE001 — telemetry must not fail maintenance
|
|
1690
|
+
pass
|
|
594
1691
|
|
|
595
1692
|
|
|
596
1693
|
def _maybe_prune_conversation_retention(
|
|
@@ -600,31 +1697,29 @@ def _maybe_prune_conversation_retention(
|
|
|
600
1697
|
retention_days: int,
|
|
601
1698
|
force: bool = False,
|
|
602
1699
|
record_phase: "Callable[[str, dict], None] | None" = None,
|
|
1700
|
+
clock: "Callable[[], float]" = time.monotonic,
|
|
603
1701
|
) -> "PruneStats | None":
|
|
604
|
-
"""
|
|
605
|
-
|
|
606
|
-
Returns the :class:`PruneStats` when a
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
non-blocking, so a
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
guarantees this — the dashboard opens a dedicated cache connection; the
|
|
626
|
-
from-zero-replay callers invoke it after releasing their sync flock, F9).
|
|
627
|
-
``retention_days <= 0`` disables retention (keep forever).
|
|
1702
|
+
"""One paced, flock-serialized retention visit (F7 + F9 + #901).
|
|
1703
|
+
|
|
1704
|
+
Returns the deletion's :class:`PruneStats` when a deletion visit ran
|
|
1705
|
+
(``complete`` False on a continuation), or ``None`` when no deletion ran
|
|
1706
|
+
(retention disabled, nothing due, the allowance in debt, a lock
|
|
1707
|
+
contended, or a reclaim-only visit). The one exception is a forced
|
|
1708
|
+
(explicit) visit that is due but finds the allowance in debt: it returns
|
|
1709
|
+
an empty ``PruneStats(complete=False)`` so the caller reports the
|
|
1710
|
+
allowance rather than mistaking the gate for lock contention.
|
|
1711
|
+
|
|
1712
|
+
Concurrency (F7) is unchanged: the MAINTENANCE flock is claimed
|
|
1713
|
+
EXCLUSIVE non-blocking and downgraded to SHARED once both provider flocks
|
|
1714
|
+
(Claude then Codex, non-blocking) are held, so a rival visit or `db
|
|
1715
|
+
vacuum` still cannot claim it while fail-closed panel readers can. Every
|
|
1716
|
+
flock is held for the whole visit. A visit with nothing due, or with the
|
|
1717
|
+
allowance in debt, takes no provider flock and writes nothing.
|
|
1718
|
+
|
|
1719
|
+
``force`` bypasses only the 24 h throttle (a from-zero replay). It never
|
|
1720
|
+
bypasses the allowance. ``conn`` must hold no provider flock and no open
|
|
1721
|
+
transaction; the visit reads through it and opens its own deletion and
|
|
1722
|
+
reclaim connections. ``retention_days <= 0`` disables retention.
|
|
628
1723
|
"""
|
|
629
1724
|
if retention_days is None or retention_days <= 0:
|
|
630
1725
|
return None
|
|
@@ -633,20 +1728,31 @@ def _maybe_prune_conversation_retention(
|
|
|
633
1728
|
core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
634
1729
|
except OSError:
|
|
635
1730
|
return None
|
|
1731
|
+
now = now_utc.astimezone(UTC)
|
|
1732
|
+
db_path = _resolve_main_db_path(conn)
|
|
636
1733
|
maint_fh = open(core.CONVERSATIONS_LOCK_MAINTENANCE_PATH, "w")
|
|
637
1734
|
try:
|
|
638
1735
|
try:
|
|
639
1736
|
fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
640
1737
|
except (BlockingIOError, OSError):
|
|
641
|
-
return None # another
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
1738
|
+
return None # another visit holds it; skip cleanly, do NOT stamp
|
|
1739
|
+
record = read_reclaim_pending(conn)
|
|
1740
|
+
_publish_record(record)
|
|
1741
|
+
state = PacingState.from_record(record, now)
|
|
1742
|
+
episode = reclaim_episode_active(store_geometry(conn), state.eligible)
|
|
1743
|
+
episode_ended = state.eligible and not episode
|
|
1744
|
+
deletion_due = (force or state.continuation_cutoff is not None
|
|
1745
|
+
or _retention_due(conn, now))
|
|
1746
|
+
reclaim_due = (episode and (state.next_attempt_at is None
|
|
1747
|
+
or now >= state.next_attempt_at)
|
|
1748
|
+
and _refusal_due(db_path, now))
|
|
1749
|
+
funded = state.available(now) >= 0
|
|
1750
|
+
# An explicit prune gated by the allowance deletes nothing; say so
|
|
1751
|
+
# (an empty continuation), never the lock-contention None.
|
|
1752
|
+
gated = PruneStats(complete=False) if (
|
|
1753
|
+
force and deletion_due and not funded) else None
|
|
1754
|
+
if not episode_ended and not (funded and (deletion_due or reclaim_due)):
|
|
1755
|
+
return gated
|
|
650
1756
|
claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
|
|
651
1757
|
try:
|
|
652
1758
|
try:
|
|
@@ -659,93 +1765,25 @@ def _maybe_prune_conversation_retention(
|
|
|
659
1765
|
fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
660
1766
|
except (BlockingIOError, OSError):
|
|
661
1767
|
return None # a Codex sync is mid-flight; retry next cycle
|
|
662
|
-
# Downgrade the maintenance flock to SHARED for the
|
|
663
|
-
#
|
|
664
|
-
#
|
|
665
|
-
# fail-CLOSED panel readers, which take this flock
|
|
666
|
-
# `LOCK_SH | LOCK_NB` and blank their column on any contention
|
|
667
|
-
# (`read_session_titles_bounded` -> every Claude session title in
|
|
668
|
-
# the Recent Sessions card, for the minutes a large prune runs).
|
|
669
|
-
# A prune is a writer, not a family replacement: concurrent
|
|
670
|
-
# writes are already excluded by the two provider flocks, and
|
|
671
|
-
# the replacement paths readers guard against take this flock
|
|
672
|
-
# EXCLUSIVE, which a held SHARED still blocks. Rival prunes and
|
|
673
|
-
# `db vacuum` claim `LOCK_EX | LOCK_NB`, which also still fails.
|
|
674
|
-
# MUST stay after both provider flocks: a flock conversion is
|
|
675
|
-
# not atomic, so a rival can slip into the downgrade window —
|
|
676
|
-
# the provider flocks it then fails to claim are what make that
|
|
677
|
-
# harmless.
|
|
1768
|
+
# Downgrade the maintenance flock to SHARED for the visit
|
|
1769
|
+
# proper, AFTER both provider flocks (a flock conversion is not
|
|
1770
|
+
# atomic; the provider flocks make the window harmless).
|
|
678
1771
|
try:
|
|
679
1772
|
fcntl.flock(maint_fh, fcntl.LOCK_SH)
|
|
680
1773
|
except OSError:
|
|
681
1774
|
pass # keep the exclusive hold; correctness is unchanged
|
|
682
|
-
if
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
cutoff = now_utc - dt.timedelta(days=int(retention_days))
|
|
695
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
696
|
-
try:
|
|
697
|
-
def commit_group() -> None:
|
|
698
|
-
conn.commit()
|
|
699
|
-
conn.execute("BEGIN IMMEDIATE")
|
|
700
|
-
|
|
701
|
-
delete_started = time.monotonic()
|
|
702
|
-
stats = prune_conversation_transcripts(
|
|
703
|
-
conn,
|
|
704
|
-
cutoff_utc=cutoff,
|
|
705
|
-
after_group=commit_group,
|
|
706
|
-
)
|
|
707
|
-
_stamp_retention_prune(conn, now_utc)
|
|
708
|
-
conn.commit()
|
|
709
|
-
except Exception:
|
|
710
|
-
conn.rollback()
|
|
711
|
-
raise
|
|
712
|
-
if record_phase is not None:
|
|
713
|
-
record_phase("delete", {
|
|
714
|
-
"duration_s": max(
|
|
715
|
-
time.monotonic() - delete_started, 0.0),
|
|
716
|
-
"claude_messages": stats.claude_messages,
|
|
717
|
-
"codex_events": stats.codex_events,
|
|
718
|
-
})
|
|
719
|
-
# Return the freed pages to the OS. On an INCREMENTAL auto-vacuum
|
|
720
|
-
# cache.db (the default for freshly-created DBs, #313 P3) this
|
|
721
|
-
# shrinks the file on disk instead of leaving a growing freelist,
|
|
722
|
-
# so the transcript prune reclaims space automatically without a
|
|
723
|
-
# manual `cctally db vacuum`; on a legacy auto_vacuum=NONE cache it
|
|
724
|
-
# is a harmless no-op (those still reclaim via `db vacuum` or a
|
|
725
|
-
# `cache-sync --rebuild`). Runs OUTSIDE the committed transaction,
|
|
726
|
-
# still under the maintenance + provider flocks (no concurrent
|
|
727
|
-
# writer), and best-effort — a reclaim error must never fail the
|
|
728
|
-
# already-durable prune.
|
|
729
|
-
# Deletion is complete and durable at this point, and only
|
|
730
|
-
# deletion completion is stamped. Reclaim is a SEPARATE phase
|
|
731
|
-
# with its own budget and its own durable pending state, so a
|
|
732
|
-
# pass that runs out of budget continues on the next cycle
|
|
733
|
-
# instead of holding three flocks until the freelist drains.
|
|
734
|
-
#
|
|
735
|
-
# A pass with no new deletion still continues an outstanding
|
|
736
|
-
# backlog: the previous implementation reclaimed only when the
|
|
737
|
-
# prune had deleted something, so a store that fell behind had
|
|
738
|
-
# no way to catch up.
|
|
739
|
-
pending_before = read_reclaim_pending(conn)
|
|
740
|
-
if stats.total_rows > 0 or pending_before is not None:
|
|
741
|
-
try:
|
|
742
|
-
phases = run_reclaim_pass(conn, now_utc=now_utc)
|
|
743
|
-
except sqlite3.Error:
|
|
744
|
-
phases = None
|
|
745
|
-
if phases is not None and record_phase is not None:
|
|
746
|
-
record_phase("reclaim", phases["reclaim"])
|
|
747
|
-
record_phase("checkpoint", phases["checkpoint"])
|
|
748
|
-
return stats
|
|
1775
|
+
if episode_ended:
|
|
1776
|
+
_end_reclaim_episode(db_path)
|
|
1777
|
+
stats = None
|
|
1778
|
+
if funded and deletion_due:
|
|
1779
|
+
stats = _run_deletion_visit(
|
|
1780
|
+
db_path, now=now, retention_days=retention_days,
|
|
1781
|
+
state=state, record_phase=record_phase, clock=clock)
|
|
1782
|
+
if funded and reclaim_due:
|
|
1783
|
+
_run_reclaim_attempt(db_path, now=now,
|
|
1784
|
+
record_phase=record_phase, clock=clock)
|
|
1785
|
+
_publish_record(read_reclaim_pending(conn))
|
|
1786
|
+
return stats if stats is not None else gated
|
|
749
1787
|
finally:
|
|
750
1788
|
try:
|
|
751
1789
|
fcntl.flock(codex_fh, fcntl.LOCK_UN)
|
|
@@ -764,76 +1802,3 @@ def _maybe_prune_conversation_retention(
|
|
|
764
1802
|
except OSError:
|
|
765
1803
|
pass
|
|
766
1804
|
maint_fh.close()
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
def _delete_codex_conversation_derived(
|
|
770
|
-
conn: sqlite3.Connection, conversation_key: str
|
|
771
|
-
) -> None:
|
|
772
|
-
"""#294 S6: drop one pruned conversation's normalized derived rows in the same
|
|
773
|
-
transaction as its physical events.
|
|
774
|
-
|
|
775
|
-
#313 prunes at WHOLE-conversation granularity — ``_prune_codex`` deletes every
|
|
776
|
-
``codex_conversation_events`` row for the key, so every event across every file
|
|
777
|
-
of the conversation is gone. The S6 rollup ownership rule (§3.2) is therefore
|
|
778
|
-
the "or-delete" branch: with zero surviving normalized rows the rollup is
|
|
779
|
-
deleted outright, no recompute needed. The normalized-message DELETE is a
|
|
780
|
-
PARTIAL delete over the whole corpus (§3.4), so it rides the per-row FTS
|
|
781
|
-
AFTER-DELETE trigger — surviving conversations keep their postings — and NEVER
|
|
782
|
-
the full-clear ``'delete-all'``; when ``codex_fts_unavailable`` is set the
|
|
783
|
-
triggers are absent and these are plain base deletes. File touches carry an
|
|
784
|
-
explicit ``conversation_key`` so they scope exactly. Deleting the derived rows
|
|
785
|
-
is part of pruning the SAME conversation, so it does not change the reported
|
|
786
|
-
``codex_conversations`` / ``codex_events`` counts (those stay physical).
|
|
787
|
-
"""
|
|
788
|
-
conn.execute(
|
|
789
|
-
"DELETE FROM codex_conversation_file_touches WHERE conversation_key = ?",
|
|
790
|
-
(conversation_key,),
|
|
791
|
-
)
|
|
792
|
-
# Rides the codex_conv_fts_ad per-row trigger (partial delete, §3.4).
|
|
793
|
-
conn.execute(
|
|
794
|
-
"DELETE FROM codex_conversation_messages WHERE conversation_key = ?",
|
|
795
|
-
(conversation_key,),
|
|
796
|
-
)
|
|
797
|
-
conn.execute(
|
|
798
|
-
"DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
|
|
799
|
-
(conversation_key,),
|
|
800
|
-
)
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
def _prune_codex(
|
|
804
|
-
conn: sqlite3.Connection,
|
|
805
|
-
cutoff: str,
|
|
806
|
-
*,
|
|
807
|
-
after_group: "Callable[[], None] | None" = None,
|
|
808
|
-
) -> tuple[int, int]:
|
|
809
|
-
conversations = 0
|
|
810
|
-
events = 0
|
|
811
|
-
for conversation_key in _prunable_groups(
|
|
812
|
-
conn, "codex_conversation_events", "conversation_key", cutoff
|
|
813
|
-
):
|
|
814
|
-
# #294 S6: purge the derived normalized rows for this conversation in the
|
|
815
|
-
# SAME transaction as the events (§3.2 or-delete + §3.4 partial delete).
|
|
816
|
-
_delete_codex_conversation_derived(conn, conversation_key)
|
|
817
|
-
cur = conn.execute(
|
|
818
|
-
"DELETE FROM codex_conversation_events WHERE conversation_key = ?",
|
|
819
|
-
(conversation_key,),
|
|
820
|
-
)
|
|
821
|
-
events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
|
|
822
|
-
conversations += 1
|
|
823
|
-
if after_group is not None:
|
|
824
|
-
after_group()
|
|
825
|
-
for source_path in _prunable_null_identity_paths(
|
|
826
|
-
conn, "codex_conversation_events", "conversation_key", cutoff
|
|
827
|
-
):
|
|
828
|
-
# NULL-conversation_key events never gained thread identity, so S6 never
|
|
829
|
-
# normalized them (§4.1) — there are no derived rows to clean up here.
|
|
830
|
-
cur = conn.execute(
|
|
831
|
-
"DELETE FROM codex_conversation_events "
|
|
832
|
-
"WHERE conversation_key IS NULL AND source_path = ?",
|
|
833
|
-
(source_path,),
|
|
834
|
-
)
|
|
835
|
-
events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
|
|
836
|
-
conversations += 1
|
|
837
|
-
if after_group is not None:
|
|
838
|
-
after_group()
|
|
839
|
-
return conversations, events
|