cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -33,7 +33,10 @@ from __future__ import annotations
|
|
|
33
33
|
|
|
34
34
|
import datetime as dt
|
|
35
35
|
import fcntl
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
36
38
|
import sqlite3
|
|
39
|
+
import time
|
|
37
40
|
from collections.abc import Callable
|
|
38
41
|
from dataclasses import dataclass
|
|
39
42
|
|
|
@@ -45,6 +48,122 @@ UTC = dt.timezone.utc
|
|
|
45
48
|
_RETENTION_LAST_PRUNE_KEY = "conversation_retention_last_prune_at"
|
|
46
49
|
_RETENTION_THROTTLE_SECONDS = 24 * 60 * 60
|
|
47
50
|
|
|
51
|
+
# --------------------------------------------------------------------------
|
|
52
|
+
# #780 — bounded, resumable reclaim
|
|
53
|
+
# --------------------------------------------------------------------------
|
|
54
|
+
|
|
55
|
+
#: Durable reclaim state (framework-untracked KV, same class as the throttle
|
|
56
|
+
#: key above — a value shape, not a schema change).
|
|
57
|
+
RECLAIM_PENDING_KEY = "conversation_retention_reclaim_pending"
|
|
58
|
+
|
|
59
|
+
#: One reclaim pass gets this much wall clock. The pass runs under the
|
|
60
|
+
#: maintenance flock and BOTH provider flocks, so an unbounded run holds all
|
|
61
|
+
#: three: the previous implementation drove the freelist to completion in
|
|
62
|
+
#: 4096-page chunks with no budget at all, and on a multi-GiB store that is
|
|
63
|
+
#: minutes of held locks against a live reader surface.
|
|
64
|
+
#:
|
|
65
|
+
#: THE DEADLINE IS CHECKED BETWEEN CHUNKS, so a pass can exceed it by at most
|
|
66
|
+
#: one chunk, and that bound is stated rather than hidden because
|
|
67
|
+
#: `PRAGMA incremental_vacuum(n)` cannot be preempted. Chunk size is the only
|
|
68
|
+
#: control, and the cost of a chunk is dominated by CACHE STATE, not by `n`.
|
|
69
|
+
#:
|
|
70
|
+
#: Measured on a copy-on-write clone of the production store, cold: 1 page
|
|
71
|
+
#: 0.006 s, 8 pages 0.001 s, 64 pages 0.009 s, 512 pages 7.754 s. An earlier
|
|
72
|
+
#: version of this comment read that series as per-page cost rising with chunk
|
|
73
|
+
#: size and concluded that "no estimator fixes this". The Tranche 3 review
|
|
74
|
+
#: disproved it and the correction matters, because the false version tells
|
|
75
|
+
#: the next maintainer not to bother measuring. Two facts settle it. Warm
|
|
76
|
+
#: passes on the SAME store sustain about 2,300 pages a second at 1,024 and
|
|
77
|
+
#: 2,048-page chunks, so a large chunk is not intrinsically expensive per page.
|
|
78
|
+
#: And the 8-page chunk being SIX TIMES cheaper than the 1-page chunk is not a
|
|
79
|
+
#: size effect at all — it is the second reading of a cold ramp warming up.
|
|
80
|
+
#: The 512-page chunk is the fourth reading of that ramp, and it is expensive
|
|
81
|
+
#: because it is the first one to reach a genuinely cold region of a multi-GiB
|
|
82
|
+
#: file, not because it asked for 512 pages.
|
|
83
|
+
#:
|
|
84
|
+
#: So the real hazard is that the ramp doubles into cold territory before it
|
|
85
|
+
#: has measured a slow chunk there — an estimator built from warm readings
|
|
86
|
+
#: cannot see the first-touch cost of pages nobody has read yet. The growth cap
|
|
87
|
+
#: below is the control that matters for exactly that reason, and a plausible
|
|
88
|
+
#: refinement, not taken here because it is unmeasured, is to require two or
|
|
89
|
+
#: three consecutive chunks at the current size before permitting a doubling,
|
|
90
|
+
#: so a size increase always rests on more than one observation.
|
|
91
|
+
#:
|
|
92
|
+
#: Observed worst case on that store: 9.9 s for the first, cold pass, then
|
|
93
|
+
#: 2.05-2.11 s for every warm pass. That is the overshoot this design accepts,
|
|
94
|
+
#: and it is bounded and reported (`deadline_hit`) rather than unbounded: the
|
|
95
|
+
#: implementation it replaces held the maintenance flock and both provider
|
|
96
|
+
#: flocks for the whole drain, which on the same store was part of a 25-minute
|
|
97
|
+
#: rebuild.
|
|
98
|
+
RECLAIM_DEADLINE_SECONDS = 2.0
|
|
99
|
+
|
|
100
|
+
#: Chunk sizing. Start small so the FIRST chunk cannot overshoot the deadline
|
|
101
|
+
#: on a cold multi-GiB file, then adapt upward from measured throughput —
|
|
102
|
+
#: DOUBLING at most, and from the SLOWEST rate this pass has seen rather than
|
|
103
|
+
#: the most recent one. Neither is a cure — see the cold-cache first-touch cost
|
|
104
|
+
#: recorded on the deadline above — but together they bound the growth to one
|
|
105
|
+
#: doubling per chunk, so the pass takes many measured steps and the single
|
|
106
|
+
#: chunk that can overshoot is at most twice the last chunk that fit inside the
|
|
107
|
+
#: budget.
|
|
108
|
+
RECLAIM_INITIAL_PAGES = 64
|
|
109
|
+
RECLAIM_MAX_PAGES = 2048
|
|
110
|
+
RECLAIM_CHUNK_GROWTH_FACTOR = 2
|
|
111
|
+
|
|
112
|
+
#: Backlog bounds. "Unbounded growth is not accepted" needs numbers, so these
|
|
113
|
+
#: are them, and each has a distinct consequence.
|
|
114
|
+
#: * above ESCALATION, reclaim stops waiting for the daily throttle and
|
|
115
|
+
#: continues on the next cycle;
|
|
116
|
+
#: * below MIN_PAGES_PER_SECOND, the pass is recorded as making no progress
|
|
117
|
+
#: and stays pending — it is never declared complete;
|
|
118
|
+
#: * above CEILING, doctor raises a FAIL and a new rebuild is refused,
|
|
119
|
+
#: because a rebuild adds staging churn to a store already failing to drain.
|
|
120
|
+
#:
|
|
121
|
+
#: THE CEILING IS SIZED FROM A MEASUREMENT, not chosen. One full rebuild of a
|
|
122
|
+
#: copy-on-write clone of the 9.2 GB production store left 1,398,528 free pages
|
|
123
|
+
#: — 5.33 GiB, 37% of the file — because reclaim is now budgeted and the
|
|
124
|
+
#: remainder is deferred. A ceiling below that would FAIL doctor and refuse the
|
|
125
|
+
#: next rebuild after every ordinary production-scale rebuild, which is a false
|
|
126
|
+
#: alarm rather than an enforcement mechanism. 16 GiB is three times the
|
|
127
|
+
#: measured single-rebuild leftover.
|
|
128
|
+
#:
|
|
129
|
+
#: The escalation threshold is what actually drains it. Measured steady rate on
|
|
130
|
+
#: the same clone: about 2,300 pages a second, so roughly 4,600 pages a 2 s
|
|
131
|
+
#: pass, so about 300 passes for that backlog. At the daily throttle that is
|
|
132
|
+
#: 300 days; at the escalated per-cycle cadence it is under an hour. That
|
|
133
|
+
#: ratio is the reason escalation exists.
|
|
134
|
+
RECLAIM_ESCALATION_BYTES = 256 * 1024 * 1024
|
|
135
|
+
RECLAIM_MIN_PAGES_PER_SECOND = 1.0
|
|
136
|
+
RECLAIM_CEILING_BYTES = 16 * 1024 * 1024 * 1024
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass(frozen=True)
|
|
140
|
+
class ReclaimOutcome:
|
|
141
|
+
"""What one bounded reclaim pass observed. Never a claim about the file."""
|
|
142
|
+
|
|
143
|
+
freelist_before: int = 0
|
|
144
|
+
freelist_after: int = 0
|
|
145
|
+
page_size: int = 0
|
|
146
|
+
pages_reclaimed: int = 0
|
|
147
|
+
duration_s: float = 0.0
|
|
148
|
+
deadline_hit: bool = False
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def unreclaimed_bytes(self) -> int:
|
|
152
|
+
return max(0, self.freelist_after) * max(0, self.page_size)
|
|
153
|
+
|
|
154
|
+
@property
|
|
155
|
+
def made_progress(self) -> bool:
|
|
156
|
+
if self.pages_reclaimed <= 0:
|
|
157
|
+
return False
|
|
158
|
+
if self.duration_s <= 0:
|
|
159
|
+
return True
|
|
160
|
+
return (self.pages_reclaimed / self.duration_s) >= (
|
|
161
|
+
RECLAIM_MIN_PAGES_PER_SECOND)
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def freelist_drained(self) -> bool:
|
|
165
|
+
return self.freelist_after <= 0
|
|
166
|
+
|
|
48
167
|
|
|
49
168
|
@dataclass(frozen=True)
|
|
50
169
|
class PruneStats:
|
|
@@ -221,25 +340,257 @@ def _stamp_retention_prune(conn: sqlite3.Connection, now_utc: dt.datetime) -> No
|
|
|
221
340
|
)
|
|
222
341
|
|
|
223
342
|
|
|
224
|
-
def
|
|
225
|
-
"""
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
343
|
+
def _run_incremental_vacuum_chunk(conn: sqlite3.Connection, pages: int) -> int:
|
|
344
|
+
"""Reclaim at most ``pages`` freelist pages; return the freelist count after.
|
|
345
|
+
|
|
346
|
+
The one seam every chunk goes through, so a test can replace it to hold the
|
|
347
|
+
real SQLite write lock at a barrier and observe what a concurrent reader
|
|
348
|
+
does. ``executescript()`` routes through ``sqlite3_exec()``, which steps
|
|
349
|
+
zero-column pragma rows through ``SQLITE_DONE`` on Python/SQLite
|
|
350
|
+
combinations where ``Cursor.fetchall()`` can stop after the first row
|
|
351
|
+
(public Linux 3.11).
|
|
352
|
+
"""
|
|
353
|
+
conn.executescript(f"PRAGMA incremental_vacuum({int(pages)});")
|
|
354
|
+
return int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _reclaim_incremental_vacuum(
|
|
358
|
+
conn: sqlite3.Connection,
|
|
359
|
+
*,
|
|
360
|
+
deadline_seconds: float = RECLAIM_DEADLINE_SECONDS,
|
|
361
|
+
clock: "Callable[[], float]" = time.monotonic,
|
|
362
|
+
) -> ReclaimOutcome:
|
|
363
|
+
"""Reclaim freelist pages under a monotonic wall-clock budget (#780).
|
|
364
|
+
|
|
365
|
+
The pass runs under the maintenance flock and both provider flocks, so its
|
|
366
|
+
cost is a cost every reader and both ingesters pay. It therefore stops at
|
|
367
|
+
the deadline and leaves the remainder for the next pass rather than driving
|
|
368
|
+
the freelist to zero however long that takes.
|
|
369
|
+
|
|
370
|
+
Chunks start small and adapt to measured throughput, so a slow disk cannot
|
|
371
|
+
overshoot the budget on the first chunk and a fast one is not held to 64
|
|
372
|
+
pages a chunk for the whole pass. The freelist is re-read after every
|
|
373
|
+
chunk, which is also how a chunk that reclaimed nothing is detected.
|
|
374
|
+
"""
|
|
375
|
+
page_size = int(conn.execute("PRAGMA page_size").fetchone()[0])
|
|
376
|
+
before = int(conn.execute("PRAGMA freelist_count").fetchone()[0])
|
|
377
|
+
started = clock()
|
|
378
|
+
if before <= 0:
|
|
379
|
+
return ReclaimOutcome(
|
|
380
|
+
freelist_before=before, freelist_after=before, page_size=page_size)
|
|
381
|
+
remaining = before
|
|
382
|
+
pages = RECLAIM_INITIAL_PAGES
|
|
383
|
+
# The SLOWEST rate observed in THIS pass, not the most recent one. A cold
|
|
384
|
+
# store's per-page cost is about twenty times its warm cost, so sizing the
|
|
385
|
+
# next chunk from the chunk just measured is optimistic exactly when it
|
|
386
|
+
# matters: measured on a clone of the production store, that estimator ran
|
|
387
|
+
# a first pass to 7.9 s against a 2.0 s budget. Taking the minimum makes
|
|
388
|
+
# one slow chunk hold the rest of the pass conservative, and the estimate
|
|
389
|
+
# resets each pass, so a warm pass still ramps.
|
|
390
|
+
slowest_rate = None
|
|
391
|
+
deadline_hit = False
|
|
392
|
+
while remaining > 0:
|
|
393
|
+
elapsed = clock() - started
|
|
394
|
+
if elapsed >= deadline_seconds:
|
|
395
|
+
deadline_hit = True
|
|
396
|
+
break
|
|
397
|
+
requested = min(remaining, pages)
|
|
398
|
+
chunk_started = clock()
|
|
399
|
+
after = _run_incremental_vacuum_chunk(conn, requested)
|
|
400
|
+
chunk_elapsed = max(clock() - chunk_started, 0.0)
|
|
240
401
|
if after >= remaining:
|
|
241
|
-
|
|
402
|
+
# The chunk reclaimed nothing. Continuing would spin against a
|
|
403
|
+
# store that is not releasing pages.
|
|
404
|
+
remaining = after
|
|
405
|
+
break
|
|
406
|
+
if chunk_elapsed > 0:
|
|
407
|
+
# Aim each chunk at a quarter of the remaining budget, so the pass
|
|
408
|
+
# takes several measured steps rather than one guess — and never
|
|
409
|
+
# more than DOUBLE the chunk just measured, so a cold first chunk
|
|
410
|
+
# cannot extrapolate the pass past its deadline.
|
|
411
|
+
rate = (remaining - after) / chunk_elapsed
|
|
412
|
+
slowest_rate = (
|
|
413
|
+
rate if slowest_rate is None else min(slowest_rate, rate))
|
|
414
|
+
budget_left = max(deadline_seconds - (clock() - started), 0.0)
|
|
415
|
+
pages = int(max(
|
|
416
|
+
RECLAIM_INITIAL_PAGES,
|
|
417
|
+
min(RECLAIM_MAX_PAGES,
|
|
418
|
+
slowest_rate * budget_left / 4,
|
|
419
|
+
requested * RECLAIM_CHUNK_GROWTH_FACTOR),
|
|
420
|
+
))
|
|
242
421
|
remaining = after
|
|
422
|
+
return ReclaimOutcome(
|
|
423
|
+
freelist_before=before,
|
|
424
|
+
freelist_after=remaining,
|
|
425
|
+
page_size=page_size,
|
|
426
|
+
pages_reclaimed=max(0, before - remaining),
|
|
427
|
+
duration_s=max(clock() - started, 0.0),
|
|
428
|
+
deadline_hit=deadline_hit,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _resolve_main_db_path(conn: sqlite3.Connection):
|
|
433
|
+
"""The file behind this connection's ``main`` schema.
|
|
434
|
+
|
|
435
|
+
Read off the connection rather than assumed, because the orchestrator is
|
|
436
|
+
called with a conversations connection by the dashboard and with a cache
|
|
437
|
+
connection by the from-zero-replay callers; measuring the wrong family's
|
|
438
|
+
size would report a reclaim against a file the pass never touched. Falls
|
|
439
|
+
back to the conversations path when the connection reports no file, which
|
|
440
|
+
is the in-memory case.
|
|
441
|
+
"""
|
|
442
|
+
try:
|
|
443
|
+
for _seq, name, file_name in conn.execute("PRAGMA database_list"):
|
|
444
|
+
if name == "main" and file_name:
|
|
445
|
+
return file_name
|
|
446
|
+
except sqlite3.Error:
|
|
447
|
+
pass
|
|
448
|
+
return _cctally_core.CONVERSATIONS_DB_PATH
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _db_family_bytes(path) -> int:
|
|
452
|
+
"""Database plus WAL plus shm, in bytes. Missing members count as zero."""
|
|
453
|
+
total = 0
|
|
454
|
+
base = str(path)
|
|
455
|
+
for member in (base, base + "-wal", base + "-shm"):
|
|
456
|
+
try:
|
|
457
|
+
total += os.path.getsize(member)
|
|
458
|
+
except OSError:
|
|
459
|
+
continue
|
|
460
|
+
return total
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _checkpoint_truncate(conn: sqlite3.Connection) -> "tuple[int, int, int] | None":
|
|
464
|
+
"""`wal_checkpoint(TRUNCATE)`, returning its `(busy, log, checkpointed)` row.
|
|
465
|
+
|
|
466
|
+
Completion is PHYSICAL, not a freelist reading. `incremental_vacuum` moves
|
|
467
|
+
pages off the freelist and drops `page_count`, but the bytes stay in the
|
|
468
|
+
WAL until it is truncated, so clearing the pending state at
|
|
469
|
+
`freelist_count == 0` would declare success while the space is still
|
|
470
|
+
occupied. Returns None when the checkpoint could not run at all.
|
|
471
|
+
"""
|
|
472
|
+
try:
|
|
473
|
+
row = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
|
474
|
+
except sqlite3.Error:
|
|
475
|
+
return None
|
|
476
|
+
if row is None:
|
|
477
|
+
return None
|
|
478
|
+
return (int(row[0]), int(row[1]), int(row[2]))
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def read_reclaim_pending(conn: sqlite3.Connection) -> "dict | None":
|
|
482
|
+
"""The durable reclaim backlog record, or None when nothing is pending."""
|
|
483
|
+
try:
|
|
484
|
+
row = conn.execute(
|
|
485
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
486
|
+
(RECLAIM_PENDING_KEY,),
|
|
487
|
+
).fetchone()
|
|
488
|
+
except sqlite3.Error:
|
|
489
|
+
return None
|
|
490
|
+
if row is None or not row[0]:
|
|
491
|
+
return None
|
|
492
|
+
try:
|
|
493
|
+
state = json.loads(row[0])
|
|
494
|
+
except (TypeError, ValueError):
|
|
495
|
+
return None
|
|
496
|
+
return state if isinstance(state, dict) else None
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _write_reclaim_pending(conn: sqlite3.Connection, state: "dict | None") -> None:
|
|
500
|
+
if state is None:
|
|
501
|
+
conn.execute("DELETE FROM cache_meta WHERE key=?", (RECLAIM_PENDING_KEY,))
|
|
502
|
+
return
|
|
503
|
+
conn.execute(
|
|
504
|
+
"INSERT INTO cache_meta(key, value) VALUES (?, ?) "
|
|
505
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
506
|
+
(RECLAIM_PENDING_KEY, json.dumps(state, sort_keys=True)),
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def reclaim_backlog_bytes(state: "dict | None") -> int:
|
|
511
|
+
"""Unreclaimed bytes a pending record reports, or 0 when nothing is."""
|
|
512
|
+
if not state:
|
|
513
|
+
return 0
|
|
514
|
+
try:
|
|
515
|
+
return max(0, int(state.get("unreclaimed_bytes") or 0))
|
|
516
|
+
except (TypeError, ValueError):
|
|
517
|
+
return 0
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def reclaim_backlog_escalated(state: "dict | None") -> bool:
|
|
521
|
+
"""Whether the backlog is large enough to stop waiting a whole day."""
|
|
522
|
+
return reclaim_backlog_bytes(state) >= RECLAIM_ESCALATION_BYTES
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def reclaim_backlog_over_ceiling(state: "dict | None") -> bool:
|
|
526
|
+
"""Whether the backlog has reached the hard ceiling: doctor FAIL, and a new
|
|
527
|
+
rebuild is refused because it would add staging churn to a store that is
|
|
528
|
+
already failing to drain."""
|
|
529
|
+
return reclaim_backlog_bytes(state) >= RECLAIM_CEILING_BYTES
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def run_reclaim_pass(
|
|
533
|
+
conn: sqlite3.Connection,
|
|
534
|
+
*,
|
|
535
|
+
now_utc: dt.datetime,
|
|
536
|
+
db_path=None,
|
|
537
|
+
deadline_seconds: float = RECLAIM_DEADLINE_SECONDS,
|
|
538
|
+
clock: "Callable[[], float]" = time.monotonic,
|
|
539
|
+
) -> "dict":
|
|
540
|
+
"""One bounded reclaim + checkpoint pass, with durable pending state.
|
|
541
|
+
|
|
542
|
+
Returns a phase record: ``{reclaim: {...}, checkpoint: {...}, pending: ...}``.
|
|
543
|
+
The pending record is retained until a MEASURED reduction in the database
|
|
544
|
+
family's size, or the checkpoint's own result, shows the space was really
|
|
545
|
+
returned. A pass that made no progress stays pending and is reported; it is
|
|
546
|
+
never declared complete.
|
|
547
|
+
"""
|
|
548
|
+
path = _resolve_main_db_path(conn) if db_path is None else db_path
|
|
549
|
+
family_before = _db_family_bytes(path)
|
|
550
|
+
outcome = _reclaim_incremental_vacuum(
|
|
551
|
+
conn, deadline_seconds=deadline_seconds, clock=clock)
|
|
552
|
+
checkpoint = _checkpoint_truncate(conn)
|
|
553
|
+
family_after = _db_family_bytes(path)
|
|
554
|
+
returned = max(0, family_before - family_after)
|
|
555
|
+
complete = (
|
|
556
|
+
outcome.freelist_drained
|
|
557
|
+
and checkpoint is not None
|
|
558
|
+
and checkpoint[0] == 0
|
|
559
|
+
and checkpoint[1] == 0
|
|
560
|
+
)
|
|
561
|
+
state = None
|
|
562
|
+
if not complete:
|
|
563
|
+
state = {
|
|
564
|
+
"freelist_count": outcome.freelist_after,
|
|
565
|
+
"unreclaimed_bytes": outcome.unreclaimed_bytes,
|
|
566
|
+
"attempted_at": now_utc.astimezone(UTC).isoformat(),
|
|
567
|
+
"made_progress": bool(outcome.made_progress),
|
|
568
|
+
"deadline_hit": bool(outcome.deadline_hit),
|
|
569
|
+
}
|
|
570
|
+
# Write and commit ONLY when the record actually changed. A pass over a
|
|
571
|
+
# store with no backlog would otherwise issue a no-op DELETE and a commit
|
|
572
|
+
# on every cycle, which is both pointless and visible to callers counting
|
|
573
|
+
# transaction boundaries.
|
|
574
|
+
if state != read_reclaim_pending(conn):
|
|
575
|
+
_write_reclaim_pending(conn, state)
|
|
576
|
+
conn.commit()
|
|
577
|
+
return {
|
|
578
|
+
"reclaim": {
|
|
579
|
+
"pages_reclaimed": outcome.pages_reclaimed,
|
|
580
|
+
"freelist_before": outcome.freelist_before,
|
|
581
|
+
"freelist_after": outcome.freelist_after,
|
|
582
|
+
"duration_s": outcome.duration_s,
|
|
583
|
+
"deadline_hit": outcome.deadline_hit,
|
|
584
|
+
"made_progress": outcome.made_progress,
|
|
585
|
+
},
|
|
586
|
+
"checkpoint": {
|
|
587
|
+
"result": checkpoint,
|
|
588
|
+
"bytes_returned": returned,
|
|
589
|
+
"family_bytes": family_after,
|
|
590
|
+
},
|
|
591
|
+
"pending": state,
|
|
592
|
+
"complete": complete,
|
|
593
|
+
}
|
|
243
594
|
|
|
244
595
|
|
|
245
596
|
def _maybe_prune_conversation_retention(
|
|
@@ -248,6 +599,7 @@ def _maybe_prune_conversation_retention(
|
|
|
248
599
|
now_utc: dt.datetime,
|
|
249
600
|
retention_days: int,
|
|
250
601
|
force: bool = False,
|
|
602
|
+
record_phase: "Callable[[str, dict], None] | None" = None,
|
|
251
603
|
) -> "PruneStats | None":
|
|
252
604
|
"""Throttled, flock-serialized transcript retention prune (F7 + F9).
|
|
253
605
|
|
|
@@ -287,8 +639,14 @@ def _maybe_prune_conversation_retention(
|
|
|
287
639
|
fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
288
640
|
except (BlockingIOError, OSError):
|
|
289
641
|
return None # another prune holds it; skip cleanly, do NOT stamp
|
|
642
|
+
# The daily throttle governs DELETION. A reclaim backlog above the
|
|
643
|
+
# escalation threshold still continues on the next cycle, because a
|
|
644
|
+
# doctor warning once a day is a report, not an enforcement mechanism.
|
|
645
|
+
reclaim_only = False
|
|
290
646
|
if not force and not _retention_due(conn, now_utc):
|
|
291
|
-
|
|
647
|
+
if not reclaim_backlog_escalated(read_reclaim_pending(conn)):
|
|
648
|
+
return None
|
|
649
|
+
reclaim_only = True
|
|
292
650
|
claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
|
|
293
651
|
try:
|
|
294
652
|
try:
|
|
@@ -321,6 +679,18 @@ def _maybe_prune_conversation_retention(
|
|
|
321
679
|
fcntl.flock(maint_fh, fcntl.LOCK_SH)
|
|
322
680
|
except OSError:
|
|
323
681
|
pass # keep the exclusive hold; correctness is unchanged
|
|
682
|
+
if reclaim_only:
|
|
683
|
+
# Deletion is throttled; the backlog is not. Nothing here
|
|
684
|
+
# stamps the throttle, so the ordinary daily deletion still
|
|
685
|
+
# happens on its own schedule.
|
|
686
|
+
try:
|
|
687
|
+
phases = run_reclaim_pass(conn, now_utc=now_utc)
|
|
688
|
+
except sqlite3.Error:
|
|
689
|
+
phases = None
|
|
690
|
+
if phases is not None and record_phase is not None:
|
|
691
|
+
record_phase("reclaim", phases["reclaim"])
|
|
692
|
+
record_phase("checkpoint", phases["checkpoint"])
|
|
693
|
+
return None
|
|
324
694
|
cutoff = now_utc - dt.timedelta(days=int(retention_days))
|
|
325
695
|
conn.execute("BEGIN IMMEDIATE")
|
|
326
696
|
try:
|
|
@@ -328,6 +698,7 @@ def _maybe_prune_conversation_retention(
|
|
|
328
698
|
conn.commit()
|
|
329
699
|
conn.execute("BEGIN IMMEDIATE")
|
|
330
700
|
|
|
701
|
+
delete_started = time.monotonic()
|
|
331
702
|
stats = prune_conversation_transcripts(
|
|
332
703
|
conn,
|
|
333
704
|
cutoff_utc=cutoff,
|
|
@@ -338,6 +709,13 @@ def _maybe_prune_conversation_retention(
|
|
|
338
709
|
except Exception:
|
|
339
710
|
conn.rollback()
|
|
340
711
|
raise
|
|
712
|
+
if record_phase is not None:
|
|
713
|
+
record_phase("delete", {
|
|
714
|
+
"duration_s": max(
|
|
715
|
+
time.monotonic() - delete_started, 0.0),
|
|
716
|
+
"claude_messages": stats.claude_messages,
|
|
717
|
+
"codex_events": stats.codex_events,
|
|
718
|
+
})
|
|
341
719
|
# Return the freed pages to the OS. On an INCREMENTAL auto-vacuum
|
|
342
720
|
# cache.db (the default for freshly-created DBs, #313 P3) this
|
|
343
721
|
# shrinks the file on disk instead of leaving a growing freelist,
|
|
@@ -348,15 +726,25 @@ def _maybe_prune_conversation_retention(
|
|
|
348
726
|
# still under the maintenance + provider flocks (no concurrent
|
|
349
727
|
# writer), and best-effort — a reclaim error must never fail the
|
|
350
728
|
# already-durable prune.
|
|
351
|
-
|
|
729
|
+
# Deletion is complete and durable at this point, and only
|
|
730
|
+
# deletion completion is stamped. Reclaim is a SEPARATE phase
|
|
731
|
+
# with its own budget and its own durable pending state, so a
|
|
732
|
+
# pass that runs out of budget continues on the next cycle
|
|
733
|
+
# instead of holding three flocks until the freelist drains.
|
|
734
|
+
#
|
|
735
|
+
# A pass with no new deletion still continues an outstanding
|
|
736
|
+
# backlog: the previous implementation reclaimed only when the
|
|
737
|
+
# prune had deleted something, so a store that fell behind had
|
|
738
|
+
# no way to catch up.
|
|
739
|
+
pending_before = read_reclaim_pending(conn)
|
|
740
|
+
if stats.total_rows > 0 or pending_before is not None:
|
|
352
741
|
try:
|
|
353
|
-
|
|
354
|
-
# bounded chunks. This clears the freelist and drops
|
|
355
|
-
# page_count; the physical file shrinks on the next
|
|
356
|
-
# `wal_checkpoint(TRUNCATE)` the sync loop forces (#297).
|
|
357
|
-
_reclaim_incremental_vacuum(conn)
|
|
742
|
+
phases = run_reclaim_pass(conn, now_utc=now_utc)
|
|
358
743
|
except sqlite3.Error:
|
|
359
|
-
|
|
744
|
+
phases = None
|
|
745
|
+
if phases is not None and record_phase is not None:
|
|
746
|
+
record_phase("reclaim", phases["reclaim"])
|
|
747
|
+
record_phase("checkpoint", phases["checkpoint"])
|
|
360
748
|
return stats
|
|
361
749
|
finally:
|
|
362
750
|
try:
|