cctally 1.112.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +4 -5
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +209 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -60,6 +60,7 @@ Spec: docs/superpowers/specs/2026-05-13-bin-cctally-split-design.md
|
|
|
60
60
|
from __future__ import annotations
|
|
61
61
|
|
|
62
62
|
import argparse
|
|
63
|
+
import contextlib
|
|
63
64
|
import contextvars
|
|
64
65
|
import datetime as dt
|
|
65
66
|
import enum
|
|
@@ -76,7 +77,7 @@ import tempfile
|
|
|
76
77
|
import time
|
|
77
78
|
import traceback
|
|
78
79
|
from dataclasses import dataclass
|
|
79
|
-
from typing import Any, Callable, NamedTuple
|
|
80
|
+
from typing import Any, Callable, Iterator, NamedTuple
|
|
80
81
|
|
|
81
82
|
|
|
82
83
|
def _cctally():
|
|
@@ -3876,13 +3877,29 @@ def _codex_quota_ledger_ddl() -> tuple[str, ...]:
|
|
|
3876
3877
|
#: because SQLite only uses an expression index when the query's expression
|
|
3877
3878
|
#: matches the indexed one — indexing the bare ``COALESCE`` while the reader
|
|
3878
3879
|
#: wraps it in ``unixepoch`` silently buys nothing.
|
|
3880
|
+
#:
|
|
3881
|
+
#: #901 Q11 (spec §5.3a, 901-PA-001 a; cache migration 048): the four order
|
|
3882
|
+
#: columns after the five equality members are the loader's ``ORDER BY
|
|
3883
|
+
#: source_root_key, captured_at_utc, resets_at_utc, source_path, line_offset``
|
|
3884
|
+
#: minus its constant first term, so each physical-group shard streams from the
|
|
3885
|
+
#: seek. Without them every shard re-sorted its group's whole history ("USE TEMP
|
|
3886
|
+
#: B-TREE FOR LAST 4 TERMS OF ORDER BY"; 3.3, 12.5 and 5.4 MB of temp in three
|
|
3887
|
+
#: measured calls), and one window group is a week of activity, not a bound.
|
|
3879
3888
|
_QUOTA_GROUP_INDEX_DDL = (
|
|
3880
3889
|
"CREATE INDEX IF NOT EXISTS idx_qws_physical_group "
|
|
3881
3890
|
"ON quota_window_snapshots("
|
|
3882
3891
|
" source_root_key, logical_limit_key, observed_slot, window_minutes,"
|
|
3883
|
-
" unixepoch(COALESCE(canonical_resets_at_utc, resets_at_utc))
|
|
3892
|
+
" unixepoch(COALESCE(canonical_resets_at_utc, resets_at_utc)),"
|
|
3893
|
+
" captured_at_utc, resets_at_utc, source_path, line_offset)"
|
|
3884
3894
|
" WHERE source='codex'"
|
|
3885
3895
|
)
|
|
3896
|
+
#: The key columns ``PRAGMA index_xinfo`` reports for the current definition
|
|
3897
|
+
#: (``None`` is the expression member). An index of the same name with any
|
|
3898
|
+
#: other key list is a pre-048 definition and is replaced.
|
|
3899
|
+
_QUOTA_GROUP_INDEX_KEY_COLUMNS = (
|
|
3900
|
+
"source_root_key", "logical_limit_key", "observed_slot", "window_minutes",
|
|
3901
|
+
None, "captured_at_utc", "resets_at_utc", "source_path", "line_offset",
|
|
3902
|
+
)
|
|
3886
3903
|
|
|
3887
3904
|
|
|
3888
3905
|
def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
|
|
@@ -3916,6 +3933,16 @@ def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
|
|
|
3916
3933
|
Guarded on ``canonical_resets_at_utc`` for the same reason the ledger is: a
|
|
3917
3934
|
legacy-shape cache whose schema apply took the FTS early-return before that
|
|
3918
3935
|
column add would raise ``no such column`` here.
|
|
3936
|
+
|
|
3937
|
+
#901 Q11 (cache migration 048) extends the key list with the loader's order
|
|
3938
|
+
columns (see ``_QUOTA_GROUP_INDEX_DDL``). ``CREATE INDEX IF NOT EXISTS``
|
|
3939
|
+
never reshapes an existing index, so an index of this name with any other
|
|
3940
|
+
key list is dropped and recreated, and the two statements commit together
|
|
3941
|
+
(its own ``BEGIN IMMEDIATE``, or a savepoint inside a caller's
|
|
3942
|
+
transaction): the loader probes for the index by name and then pins it with
|
|
3943
|
+
``INDEXED BY``, so a reader must never find it missing in between. A
|
|
3944
|
+
current index is left alone, so the helper is idempotent and costs one
|
|
3945
|
+
``PRAGMA`` on a current store.
|
|
3919
3946
|
"""
|
|
3920
3947
|
cols = {
|
|
3921
3948
|
str(row[1]) for row in conn.execute(
|
|
@@ -3923,8 +3950,34 @@ def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
|
|
|
3923
3950
|
}
|
|
3924
3951
|
if not cols or "canonical_resets_at_utc" not in cols:
|
|
3925
3952
|
return
|
|
3926
|
-
|
|
3927
|
-
|
|
3953
|
+
current = tuple(
|
|
3954
|
+
row[2] for row in conn.execute(
|
|
3955
|
+
"PRAGMA index_xinfo(idx_qws_physical_group)") if row[5])
|
|
3956
|
+
retired = conn.execute(
|
|
3957
|
+
"SELECT 1 FROM sqlite_master WHERE type='index'"
|
|
3958
|
+
" AND name='idx_qws_window_ident'").fetchone() is not None
|
|
3959
|
+
if current == _QUOTA_GROUP_INDEX_KEY_COLUMNS and not retired:
|
|
3960
|
+
return
|
|
3961
|
+
own_transaction = not conn.in_transaction
|
|
3962
|
+
conn.execute(
|
|
3963
|
+
"BEGIN IMMEDIATE" if own_transaction
|
|
3964
|
+
else "SAVEPOINT codex_quota_group_index")
|
|
3965
|
+
try:
|
|
3966
|
+
if current and current != _QUOTA_GROUP_INDEX_KEY_COLUMNS:
|
|
3967
|
+
conn.execute("DROP INDEX IF EXISTS idx_qws_physical_group")
|
|
3968
|
+
conn.execute(_QUOTA_GROUP_INDEX_DDL)
|
|
3969
|
+
conn.execute("DROP INDEX IF EXISTS idx_qws_window_ident")
|
|
3970
|
+
except BaseException:
|
|
3971
|
+
if own_transaction:
|
|
3972
|
+
conn.rollback()
|
|
3973
|
+
else:
|
|
3974
|
+
conn.execute("ROLLBACK TO codex_quota_group_index")
|
|
3975
|
+
conn.execute("RELEASE codex_quota_group_index")
|
|
3976
|
+
raise
|
|
3977
|
+
if own_transaction:
|
|
3978
|
+
conn.commit()
|
|
3979
|
+
else:
|
|
3980
|
+
conn.execute("RELEASE codex_quota_group_index")
|
|
3928
3981
|
|
|
3929
3982
|
|
|
3930
3983
|
#: Partial index over the rows the standing `observed_model` resolution can
|
|
@@ -4283,6 +4336,410 @@ def _apply_codex_accounting_change_ledger(conn: sqlite3.Connection) -> None:
|
|
|
4283
4336
|
conn.execute(statement)
|
|
4284
4337
|
|
|
4285
4338
|
|
|
4339
|
+
# ── #901 W1: the maintained raw-partition maxima (spec §5.1, T1) ─────────────
|
|
4340
|
+
#
|
|
4341
|
+
# `load_codex_quota_observations(latest_per_identity=True)` computed
|
|
4342
|
+
# `MAX(unixepoch(captured_at_utc)) OVER (PARTITION BY <nine raw columns>)` across
|
|
4343
|
+
# every retained Codex quota row on every doctor refresh; on the measured store
|
|
4344
|
+
# that sorted 565K rows through ~1.1 GB of temp files per call (spec §1.2). The
|
|
4345
|
+
# table below holds exactly that window's per-partition maximum, maintained by
|
|
4346
|
+
# triggers in the same transaction as every mutation, so the latest read becomes
|
|
4347
|
+
# a scan of ~10^3 summary rows plus one index seek per partition for the rows
|
|
4348
|
+
# tied at its maximum second (tests/test_901_partition_latest_maintenance.py
|
|
4349
|
+
# holds it to the frozen window query after every writer class).
|
|
4350
|
+
#
|
|
4351
|
+
# Keys are `quote(<raw column>)`: NOT NULL, collision-free across NULL, '' and
|
|
4352
|
+
# storage classes, and equal exactly when the window's PARTITION BY puts two
|
|
4353
|
+
# rows in one partition (no collation on these columns; TEXT affinity stores
|
|
4354
|
+
# numbers as text and INTEGER affinity integral reals as integers, so value
|
|
4355
|
+
# equality and quote() equality coincide). The key columns are declared BLOB on
|
|
4356
|
+
# purpose: a TEXT key column gives `quote(q.col) = s.key` TEXT comparison
|
|
4357
|
+
# affinity, and SQLite then refuses the expression index (measured: the tie
|
|
4358
|
+
# fetch fell back to walking the whole `source` autoindex). The seek index leads
|
|
4359
|
+
# with `source` for the same measured reason.
|
|
4360
|
+
#
|
|
4361
|
+
# The triggers use built-in functions only (quote, trim, coalesce, unixepoch),
|
|
4362
|
+
# so an older binary that opens the store keeps them firing. They never upsert:
|
|
4363
|
+
# an outer statement's conflict policy propagates into trigger-body statements,
|
|
4364
|
+
# so every repair is a DELETE followed by an INSERT that cannot conflict. A
|
|
4365
|
+
# REPLACE against the snapshots table would delete rows without firing the
|
|
4366
|
+
# delete trigger (recursive_triggers is off); no writer issues one, and
|
|
4367
|
+
# test_no_writer_replaces_quota_rows fails if one appears.
|
|
4368
|
+
#
|
|
4369
|
+
# `unixepoch()` in an index expression throws on the literal 'now', so the seek
|
|
4370
|
+
# index refuses a Codex row whose capture is 'now' — the same property
|
|
4371
|
+
# idx_qws_physical_group already has for the reset. Ingest writes only
|
|
4372
|
+
# `_utc_iso` instants.
|
|
4373
|
+
|
|
4374
|
+
#: Must equal `_cctally_quota._CODEX_QUOTA_REQUIRED_TEXT` (#500: the loader and
|
|
4375
|
+
#: every population derived for it agree on one required-text set);
|
|
4376
|
+
#: tests/test_901_partition_latest_maintenance.py pins the equality.
|
|
4377
|
+
_CODEX_QUOTA_LATEST_REQUIRED_TEXT = (
|
|
4378
|
+
"source", "source_root_key", "source_path", "captured_at_utc",
|
|
4379
|
+
"observed_slot", "logical_limit_key", "resets_at_utc",
|
|
4380
|
+
)
|
|
4381
|
+
|
|
4382
|
+
#: (summary key column, raw column; None means the coalesced reset anchor), in
|
|
4383
|
+
#: the window query's PARTITION BY order.
|
|
4384
|
+
_CODEX_QUOTA_LATEST_KEYS = (
|
|
4385
|
+
("k_source_root_key", "source_root_key"),
|
|
4386
|
+
("k_logical_limit_key", "logical_limit_key"),
|
|
4387
|
+
("k_observed_slot", "observed_slot"),
|
|
4388
|
+
("k_window_minutes", "window_minutes"),
|
|
4389
|
+
("k_limit_id", "limit_id"),
|
|
4390
|
+
("k_limit_name", "limit_name"),
|
|
4391
|
+
("k_observed_model", "observed_model"),
|
|
4392
|
+
("k_account_key", "account_key"),
|
|
4393
|
+
("k_reset_anchor", None),
|
|
4394
|
+
)
|
|
4395
|
+
_CODEX_QUOTA_LATEST_KEY_COLUMNS = tuple(
|
|
4396
|
+
key for key, _column in _CODEX_QUOTA_LATEST_KEYS)
|
|
4397
|
+
|
|
4398
|
+
#: Every column a partition key or a validity predicate reads.
|
|
4399
|
+
_CODEX_QUOTA_LATEST_UPDATE_OF = (
|
|
4400
|
+
"source", "source_root_key", "source_path", "captured_at_utc",
|
|
4401
|
+
"observed_slot", "logical_limit_key", "resets_at_utc",
|
|
4402
|
+
"canonical_resets_at_utc", "window_minutes", "limit_id", "limit_name",
|
|
4403
|
+
"observed_model", "account_key",
|
|
4404
|
+
)
|
|
4405
|
+
|
|
4406
|
+
#: The five objects whose joint presence means the summary has been maintained
|
|
4407
|
+
#: since it was bootstrapped. The reader uses it only when all five exist.
|
|
4408
|
+
_CODEX_QUOTA_LATEST_OBJECTS = (
|
|
4409
|
+
("table", "codex_quota_partition_latest"),
|
|
4410
|
+
("index", "idx_qws_partition_capture"),
|
|
4411
|
+
("trigger", "trg_qws_latest_ins"),
|
|
4412
|
+
("trigger", "trg_qws_latest_del"),
|
|
4413
|
+
("trigger", "trg_qws_latest_upd"),
|
|
4414
|
+
)
|
|
4415
|
+
|
|
4416
|
+
|
|
4417
|
+
def _codex_quota_latest_raw(column, prefix: str) -> str:
|
|
4418
|
+
if column is None:
|
|
4419
|
+
return (f"COALESCE({prefix}canonical_resets_at_utc, "
|
|
4420
|
+
f"{prefix}resets_at_utc)")
|
|
4421
|
+
return f"{prefix}{column}"
|
|
4422
|
+
|
|
4423
|
+
|
|
4424
|
+
def _codex_quota_latest_validity_sql(prefix: str = "") -> str:
|
|
4425
|
+
"""The loader's SQL-statable validity predicates, for one row image."""
|
|
4426
|
+
terms = [f"{prefix}source='codex'", f"{prefix}source_root_key IS NOT NULL"]
|
|
4427
|
+
terms += [f"trim(coalesce({prefix}{column}, '')) <> ''"
|
|
4428
|
+
for column in _CODEX_QUOTA_LATEST_REQUIRED_TEXT]
|
|
4429
|
+
terms += [f"unixepoch({prefix}captured_at_utc) IS NOT NULL",
|
|
4430
|
+
f"unixepoch({prefix}resets_at_utc) IS NOT NULL"]
|
|
4431
|
+
return " AND ".join(terms)
|
|
4432
|
+
|
|
4433
|
+
|
|
4434
|
+
def _codex_quota_latest_key_exprs(prefix: str = "") -> "tuple[str, ...]":
|
|
4435
|
+
return tuple(
|
|
4436
|
+
f"quote({_codex_quota_latest_raw(column, prefix)})"
|
|
4437
|
+
for _key, column in _CODEX_QUOTA_LATEST_KEYS)
|
|
4438
|
+
|
|
4439
|
+
|
|
4440
|
+
def _codex_quota_latest_summary_match(image: str) -> str:
|
|
4441
|
+
return " AND ".join(
|
|
4442
|
+
f"{key} = {expr}" for key, expr in zip(
|
|
4443
|
+
_CODEX_QUOTA_LATEST_KEY_COLUMNS,
|
|
4444
|
+
_codex_quota_latest_key_exprs(f"{image}.")))
|
|
4445
|
+
|
|
4446
|
+
|
|
4447
|
+
def _codex_quota_latest_base_match(image: str) -> str:
|
|
4448
|
+
return " AND ".join(
|
|
4449
|
+
f"{base} = {expr}" for base, expr in zip(
|
|
4450
|
+
_codex_quota_latest_key_exprs(""),
|
|
4451
|
+
_codex_quota_latest_key_exprs(f"{image}.")))
|
|
4452
|
+
|
|
4453
|
+
|
|
4454
|
+
def _codex_quota_latest_recompute_sql(image: str, *, only_if_absent: bool) -> str:
|
|
4455
|
+
"""Re-derive one partition's maximum from the physical rows (one seek)."""
|
|
4456
|
+
absent_clause = (
|
|
4457
|
+
" AND NOT EXISTS (SELECT 1 FROM codex_quota_partition_latest"
|
|
4458
|
+
f" WHERE {_codex_quota_latest_summary_match(image)})"
|
|
4459
|
+
if only_if_absent else ""
|
|
4460
|
+
)
|
|
4461
|
+
return (
|
|
4462
|
+
"INSERT INTO codex_quota_partition_latest ("
|
|
4463
|
+
+ ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
|
|
4464
|
+
+ ", latest_capture_epoch)"
|
|
4465
|
+
f" SELECT {', '.join(_codex_quota_latest_key_exprs(image + '.'))},"
|
|
4466
|
+
" latest FROM (SELECT MAX(unixepoch(captured_at_utc)) AS latest"
|
|
4467
|
+
" FROM quota_window_snapshots"
|
|
4468
|
+
f" WHERE {_codex_quota_latest_validity_sql('')}"
|
|
4469
|
+
f" AND {_codex_quota_latest_base_match(image)})"
|
|
4470
|
+
f" WHERE latest IS NOT NULL{absent_clause}"
|
|
4471
|
+
)
|
|
4472
|
+
|
|
4473
|
+
|
|
4474
|
+
def _codex_quota_latest_bootstrap_select() -> str:
|
|
4475
|
+
"""Every valid raw partition's maximum second, streamed off the seek index."""
|
|
4476
|
+
keys = ", ".join(_codex_quota_latest_key_exprs(""))
|
|
4477
|
+
return (
|
|
4478
|
+
f"SELECT {keys}, MAX(unixepoch(captured_at_utc))"
|
|
4479
|
+
" FROM quota_window_snapshots INDEXED BY idx_qws_partition_capture"
|
|
4480
|
+
f" WHERE {_codex_quota_latest_validity_sql('')}"
|
|
4481
|
+
f" GROUP BY source, {keys}"
|
|
4482
|
+
)
|
|
4483
|
+
|
|
4484
|
+
|
|
4485
|
+
def _codex_quota_latest_bootstrap_sql() -> str:
|
|
4486
|
+
return (
|
|
4487
|
+
"INSERT INTO codex_quota_partition_latest ("
|
|
4488
|
+
+ ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
|
|
4489
|
+
+ ", latest_capture_epoch) "
|
|
4490
|
+
+ _codex_quota_latest_bootstrap_select()
|
|
4491
|
+
)
|
|
4492
|
+
|
|
4493
|
+
|
|
4494
|
+
def _codex_quota_latest_ddl() -> "tuple[str, ...]":
|
|
4495
|
+
"""DDL for the summary, its seek index and its three triggers.
|
|
4496
|
+
|
|
4497
|
+
The triggers are dropped and re-created on every apply, like the change
|
|
4498
|
+
ledger's, so a definition change never leaves an older body behind; the
|
|
4499
|
+
table keeps IF NOT EXISTS and its rows survive.
|
|
4500
|
+
"""
|
|
4501
|
+
columns = ",\n".join(
|
|
4502
|
+
f" {key} BLOB NOT NULL"
|
|
4503
|
+
for key in _CODEX_QUOTA_LATEST_KEY_COLUMNS)
|
|
4504
|
+
keys = ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
|
|
4505
|
+
index_columns = ", ".join((
|
|
4506
|
+
"source", *_codex_quota_latest_key_exprs(""),
|
|
4507
|
+
"unixepoch(captured_at_utc)"))
|
|
4508
|
+
insert_new = (
|
|
4509
|
+
f"INSERT INTO codex_quota_partition_latest ({keys}, latest_capture_epoch)"
|
|
4510
|
+
f" SELECT {', '.join(_codex_quota_latest_key_exprs('NEW.'))},"
|
|
4511
|
+
" unixepoch(NEW.captured_at_utc)"
|
|
4512
|
+
" WHERE NOT EXISTS (SELECT 1 FROM codex_quota_partition_latest"
|
|
4513
|
+
f" WHERE {_codex_quota_latest_summary_match('NEW')})"
|
|
4514
|
+
)
|
|
4515
|
+
return (
|
|
4516
|
+
"CREATE TABLE IF NOT EXISTS codex_quota_partition_latest (\n"
|
|
4517
|
+
f"{columns},\n"
|
|
4518
|
+
" latest_capture_epoch BLOB NOT NULL,\n"
|
|
4519
|
+
f" PRIMARY KEY ({keys})\n"
|
|
4520
|
+
" ) WITHOUT ROWID",
|
|
4521
|
+
"CREATE INDEX IF NOT EXISTS idx_qws_partition_capture "
|
|
4522
|
+
f"ON quota_window_snapshots({index_columns}) "
|
|
4523
|
+
f"WHERE {_codex_quota_latest_validity_sql('')}",
|
|
4524
|
+
"DROP TRIGGER IF EXISTS trg_qws_latest_ins",
|
|
4525
|
+
"CREATE TRIGGER trg_qws_latest_ins\n"
|
|
4526
|
+
" AFTER INSERT ON quota_window_snapshots\n"
|
|
4527
|
+
f" WHEN {_codex_quota_latest_validity_sql('NEW.')}\n"
|
|
4528
|
+
" BEGIN\n"
|
|
4529
|
+
" DELETE FROM codex_quota_partition_latest"
|
|
4530
|
+
f" WHERE {_codex_quota_latest_summary_match('NEW')}"
|
|
4531
|
+
" AND latest_capture_epoch < unixepoch(NEW.captured_at_utc);\n"
|
|
4532
|
+
f" {insert_new};\n"
|
|
4533
|
+
" END",
|
|
4534
|
+
"DROP TRIGGER IF EXISTS trg_qws_latest_del",
|
|
4535
|
+
"CREATE TRIGGER trg_qws_latest_del\n"
|
|
4536
|
+
" AFTER DELETE ON quota_window_snapshots\n"
|
|
4537
|
+
f" WHEN {_codex_quota_latest_validity_sql('OLD.')}\n"
|
|
4538
|
+
" BEGIN\n"
|
|
4539
|
+
" DELETE FROM codex_quota_partition_latest"
|
|
4540
|
+
f" WHERE {_codex_quota_latest_summary_match('OLD')}"
|
|
4541
|
+
" AND latest_capture_epoch = unixepoch(OLD.captured_at_utc);\n"
|
|
4542
|
+
" "
|
|
4543
|
+
f"{_codex_quota_latest_recompute_sql('OLD', only_if_absent=True)};\n"
|
|
4544
|
+
" END",
|
|
4545
|
+
"DROP TRIGGER IF EXISTS trg_qws_latest_upd",
|
|
4546
|
+
"CREATE TRIGGER trg_qws_latest_upd\n"
|
|
4547
|
+
f" AFTER UPDATE OF {', '.join(_CODEX_QUOTA_LATEST_UPDATE_OF)}"
|
|
4548
|
+
" ON quota_window_snapshots\n"
|
|
4549
|
+
" WHEN OLD.source = 'codex' OR NEW.source = 'codex'\n"
|
|
4550
|
+
" BEGIN\n"
|
|
4551
|
+
" DELETE FROM codex_quota_partition_latest"
|
|
4552
|
+
f" WHERE {_codex_quota_latest_summary_match('OLD')};\n"
|
|
4553
|
+
" "
|
|
4554
|
+
f"{_codex_quota_latest_recompute_sql('OLD', only_if_absent=False)};\n"
|
|
4555
|
+
" DELETE FROM codex_quota_partition_latest"
|
|
4556
|
+
f" WHERE {_codex_quota_latest_summary_match('NEW')};\n"
|
|
4557
|
+
" "
|
|
4558
|
+
f"{_codex_quota_latest_recompute_sql('NEW', only_if_absent=False)};\n"
|
|
4559
|
+
" END",
|
|
4560
|
+
)
|
|
4561
|
+
|
|
4562
|
+
|
|
4563
|
+
def _apply_codex_quota_latest_summary(
|
|
4564
|
+
conn: sqlite3.Connection, *, rebootstrap: bool = False,
|
|
4565
|
+
) -> None:
|
|
4566
|
+
"""Install the latest-capture summary and keep it equal to the physical rows.
|
|
4567
|
+
|
|
4568
|
+
Atomic: the table, its seek index, the three triggers and, when needed, the
|
|
4569
|
+
bootstrap commit together — as a savepoint inside the caller's open
|
|
4570
|
+
transaction, or as an IMMEDIATE transaction of their own. The bootstrap runs
|
|
4571
|
+
when any of the five objects was missing before this call (so a store that
|
|
4572
|
+
carries all five has been maintained by the triggers since they were
|
|
4573
|
+
created) or when ``rebootstrap`` asks for it, which migration 047 always
|
|
4574
|
+
does, so a re-upgrade after an older binary trimmed its marker re-derives
|
|
4575
|
+
the summary from the physical rows.
|
|
4576
|
+
|
|
4577
|
+
Guarded on the three columns the partition reads: a legacy-shape cache whose
|
|
4578
|
+
schema apply took the FTS early-return before they existed keeps the
|
|
4579
|
+
window-query fallback, which is correct, just not spill-free.
|
|
4580
|
+
"""
|
|
4581
|
+
columns = {
|
|
4582
|
+
str(row[1]) for row in conn.execute(
|
|
4583
|
+
"PRAGMA table_info(quota_window_snapshots)")
|
|
4584
|
+
}
|
|
4585
|
+
if not {"canonical_resets_at_utc", "observed_model", "account_key"} <= columns:
|
|
4586
|
+
return
|
|
4587
|
+
own_transaction = not conn.in_transaction
|
|
4588
|
+
conn.execute(
|
|
4589
|
+
"BEGIN IMMEDIATE" if own_transaction
|
|
4590
|
+
else "SAVEPOINT codex_quota_latest_apply")
|
|
4591
|
+
try:
|
|
4592
|
+
present = {
|
|
4593
|
+
(str(row[0]), str(row[1])) for row in conn.execute(
|
|
4594
|
+
"SELECT type, name FROM sqlite_master "
|
|
4595
|
+
"WHERE type IN ('table', 'index', 'trigger')")
|
|
4596
|
+
}
|
|
4597
|
+
complete = all(item in present for item in _CODEX_QUOTA_LATEST_OBJECTS)
|
|
4598
|
+
for statement in _codex_quota_latest_ddl():
|
|
4599
|
+
conn.execute(statement)
|
|
4600
|
+
if rebootstrap or not complete:
|
|
4601
|
+
conn.execute("DELETE FROM codex_quota_partition_latest")
|
|
4602
|
+
conn.execute(_codex_quota_latest_bootstrap_sql())
|
|
4603
|
+
except BaseException:
|
|
4604
|
+
if own_transaction:
|
|
4605
|
+
conn.rollback()
|
|
4606
|
+
else:
|
|
4607
|
+
conn.execute("ROLLBACK TO codex_quota_latest_apply")
|
|
4608
|
+
conn.execute("RELEASE codex_quota_latest_apply")
|
|
4609
|
+
raise
|
|
4610
|
+
if own_transaction:
|
|
4611
|
+
conn.commit()
|
|
4612
|
+
else:
|
|
4613
|
+
conn.execute("RELEASE codex_quota_latest_apply")
|
|
4614
|
+
|
|
4615
|
+
|
|
4616
|
+
def _apply_codex_quota_load_order_index(conn: sqlite3.Connection) -> None:
|
|
4617
|
+
"""The general loader's ORDER BY, served by an index (#901 §5.1, T1/T3).
|
|
4618
|
+
|
|
4619
|
+
Adoption (W3a, every Codex ingest including the hook), breakdown evidence,
|
|
4620
|
+
the projection's root loads and the CLI commands all read complete
|
|
4621
|
+
populations in ``source_root_key, captured_at_utc, resets_at_utc,
|
|
4622
|
+
source_path, line_offset`` order; without this index each one sorted its
|
|
4623
|
+
whole population through a temp b-tree. ``canonical_resets_at_utc`` rides
|
|
4624
|
+
at the end so the adoption pass's canonical-reset bound is evaluated on the
|
|
4625
|
+
index entry before any table row is fetched. Guarded on that column like
|
|
4626
|
+
idx_qws_physical_group.
|
|
4627
|
+
"""
|
|
4628
|
+
columns = {
|
|
4629
|
+
str(row[1]) for row in conn.execute(
|
|
4630
|
+
"PRAGMA table_info(quota_window_snapshots)")
|
|
4631
|
+
}
|
|
4632
|
+
if "canonical_resets_at_utc" not in columns:
|
|
4633
|
+
return
|
|
4634
|
+
conn.execute(
|
|
4635
|
+
"CREATE INDEX IF NOT EXISTS idx_qws_codex_load_order "
|
|
4636
|
+
"ON quota_window_snapshots(source_root_key, captured_at_utc, "
|
|
4637
|
+
"resets_at_utc, source_path, line_offset, canonical_resets_at_utc) "
|
|
4638
|
+
"WHERE source='codex'"
|
|
4639
|
+
)
|
|
4640
|
+
|
|
4641
|
+
|
|
4642
|
+
def _apply_codex_metadata_lookup_indexes(conn: sqlite3.Connection) -> None:
|
|
4643
|
+
"""The W2 correlated lookups' indexes (#901 §5.2, T2).
|
|
4644
|
+
|
|
4645
|
+
``idx_codex_entries_root_path_time`` answers each file's and each thread's
|
|
4646
|
+
first accounting instant with one covering seek (the existing
|
|
4647
|
+
``idx_codex_entries_root_path`` keeps serving the minimum ``id``, because
|
|
4648
|
+
the rowid trails its two columns). ``idx_codex_files_alias_recent`` and
|
|
4649
|
+
``idx_codex_threads_recent`` hold the two queries' final orders so neither
|
|
4650
|
+
sorts. Each is guarded on its columns, because a legacy-shape cache can
|
|
4651
|
+
lack the ALTER-added ones.
|
|
4652
|
+
"""
|
|
4653
|
+
def _columns(table: str) -> "set[str]":
|
|
4654
|
+
return {
|
|
4655
|
+
str(row[0]) for row in conn.execute(
|
|
4656
|
+
"SELECT name FROM pragma_table_info(?)", (table,))
|
|
4657
|
+
}
|
|
4658
|
+
|
|
4659
|
+
if {"source_root_key", "source_path", "timestamp_utc"} <= _columns(
|
|
4660
|
+
"codex_session_entries"):
|
|
4661
|
+
conn.execute(
|
|
4662
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_entries_root_path_time "
|
|
4663
|
+
"ON codex_session_entries(source_root_key, source_path, "
|
|
4664
|
+
"timestamp_utc)"
|
|
4665
|
+
)
|
|
4666
|
+
if {"last_ingested_at", "path", "last_native_thread_id"} <= _columns(
|
|
4667
|
+
"codex_session_files"):
|
|
4668
|
+
conn.execute(
|
|
4669
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_files_alias_recent "
|
|
4670
|
+
"ON codex_session_files(last_ingested_at, path) "
|
|
4671
|
+
"WHERE last_native_thread_id IS NOT NULL "
|
|
4672
|
+
"AND last_native_thread_id != ''"
|
|
4673
|
+
)
|
|
4674
|
+
if {"last_seen_utc", "conversation_key"} <= _columns(
|
|
4675
|
+
"codex_conversation_threads"):
|
|
4676
|
+
conn.execute(
|
|
4677
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_threads_recent "
|
|
4678
|
+
"ON codex_conversation_threads(last_seen_utc, conversation_key)"
|
|
4679
|
+
)
|
|
4680
|
+
|
|
4681
|
+
|
|
4682
|
+
def _apply_pricing_model_indexes(conn: sqlite3.Connection) -> None:
|
|
4683
|
+
"""Model-first covering indexes for ``_pricing_observed_models`` (#901 §5.3).
|
|
4684
|
+
|
|
4685
|
+
Both of its modes — doctor's trailing 30 days and pricing-check's whole
|
|
4686
|
+
history — group by model over one entries table. Covering the token columns
|
|
4687
|
+
lets SQLite stream the GROUP BY off the index with the timestamp filter
|
|
4688
|
+
evaluated on the entry; a narrower ``(model, timestamp_utc)`` index streams
|
|
4689
|
+
too but pays one table lookup per row, a latency regression the §6.3
|
|
4690
|
+
per-call-site gate would catch.
|
|
4691
|
+
"""
|
|
4692
|
+
def _columns(table: str) -> "set[str]":
|
|
4693
|
+
return {
|
|
4694
|
+
str(row[0]) for row in conn.execute(
|
|
4695
|
+
"SELECT name FROM pragma_table_info(?)", (table,))
|
|
4696
|
+
}
|
|
4697
|
+
|
|
4698
|
+
if {"model", "timestamp_utc", "input_tokens", "output_tokens",
|
|
4699
|
+
"cache_create_tokens", "cache_read_tokens"} <= _columns(
|
|
4700
|
+
"session_entries"):
|
|
4701
|
+
conn.execute(
|
|
4702
|
+
"CREATE INDEX IF NOT EXISTS idx_entries_model_time "
|
|
4703
|
+
"ON session_entries(model, timestamp_utc, input_tokens, "
|
|
4704
|
+
"output_tokens, cache_create_tokens, cache_read_tokens)"
|
|
4705
|
+
)
|
|
4706
|
+
if {"model", "timestamp_utc", "total_tokens"} <= _columns(
|
|
4707
|
+
"codex_session_entries"):
|
|
4708
|
+
conn.execute(
|
|
4709
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_entries_model_time "
|
|
4710
|
+
"ON codex_session_entries(model, timestamp_utc, total_tokens)"
|
|
4711
|
+
)
|
|
4712
|
+
|
|
4713
|
+
|
|
4714
|
+
def _apply_codex_window_attribution_read_index(conn: sqlite3.Connection) -> None:
|
|
4715
|
+
"""The window-attribution read's order, served by an index (#901 I1/G1).
|
|
4716
|
+
|
|
4717
|
+
``_cctally_cache.load_active_window_attributions`` (every Codex ingest's
|
|
4718
|
+
spend reconciliation, so the dashboard and the Codex hook, and the
|
|
4719
|
+
attribution overlay of every quota load) returns every asserted or
|
|
4720
|
+
retracted weekly attribution in ``asserted_at_utc, op_id`` order; without
|
|
4721
|
+
this index it sorted all of them through a temp b-tree, a population that
|
|
4722
|
+
grows by one assertion per attributed window. ``window_minutes`` leads
|
|
4723
|
+
because the read's one equality filter is on it; the root, retraction and
|
|
4724
|
+
account predicates are evaluated along the ordered scan. ``source_root_key``
|
|
4725
|
+
is deliberately NOT ahead of the ordering columns: several roots share one
|
|
4726
|
+
global assertion order (Amendment 1 item 1, ``c901-design-2``). Guarded on
|
|
4727
|
+
the table's columns, because the conversations store drops the table and a
|
|
4728
|
+
legacy-shape cache can predate it.
|
|
4729
|
+
"""
|
|
4730
|
+
columns = {
|
|
4731
|
+
str(row[0]) for row in conn.execute(
|
|
4732
|
+
"SELECT name FROM pragma_table_info(?)",
|
|
4733
|
+
("codex_window_attributions",))
|
|
4734
|
+
}
|
|
4735
|
+
if {"window_minutes", "asserted_at_utc", "op_id"} <= columns:
|
|
4736
|
+
conn.execute(
|
|
4737
|
+
"CREATE INDEX IF NOT EXISTS idx_codex_window_attributions_read_order "
|
|
4738
|
+
"ON codex_window_attributions(window_minutes ASC, asserted_at_utc ASC, "
|
|
4739
|
+
"op_id ASC)"
|
|
4740
|
+
)
|
|
4741
|
+
|
|
4742
|
+
|
|
4286
4743
|
# === Region 7b2: Eager cache-migration trigger (V4 — same-invocation 008 apply) ===
|
|
4287
4744
|
|
|
4288
4745
|
|
|
@@ -4451,7 +4908,7 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
4451
4908
|
"031_codex_file_account_map"),
|
|
4452
4909
|
SchemaDeliveryObject(
|
|
4453
4910
|
"index", "idx_qws_physical_group", "_apply_codex_quota_group_index",
|
|
4454
|
-
"
|
|
4911
|
+
"048_codex_quota_physical_group_order"),
|
|
4455
4912
|
SchemaDeliveryObject(
|
|
4456
4913
|
"index", "idx_qws_unresolved_model",
|
|
4457
4914
|
"_apply_codex_quota_unresolved_model_index",
|
|
@@ -4537,6 +4994,43 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
4537
4994
|
"trigger", "trg_codex_accounting_upd",
|
|
4538
4995
|
"_apply_codex_accounting_change_ledger",
|
|
4539
4996
|
"044_codex_accounting_change_ledger"),
|
|
4997
|
+
SchemaDeliveryObject(
|
|
4998
|
+
"table", "codex_quota_partition_latest",
|
|
4999
|
+
"_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
|
|
5000
|
+
SchemaDeliveryObject(
|
|
5001
|
+
"index", "idx_qws_partition_capture",
|
|
5002
|
+
"_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
|
|
5003
|
+
SchemaDeliveryObject(
|
|
5004
|
+
"trigger", "trg_qws_latest_del",
|
|
5005
|
+
"_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
|
|
5006
|
+
SchemaDeliveryObject(
|
|
5007
|
+
"trigger", "trg_qws_latest_ins",
|
|
5008
|
+
"_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
|
|
5009
|
+
SchemaDeliveryObject(
|
|
5010
|
+
"trigger", "trg_qws_latest_upd",
|
|
5011
|
+
"_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
|
|
5012
|
+
SchemaDeliveryObject(
|
|
5013
|
+
"index", "idx_qws_codex_load_order",
|
|
5014
|
+
"_apply_codex_quota_load_order_index", "047_spill_free_read_paths"),
|
|
5015
|
+
SchemaDeliveryObject(
|
|
5016
|
+
"index", "idx_codex_entries_root_path_time",
|
|
5017
|
+
"_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
|
|
5018
|
+
SchemaDeliveryObject(
|
|
5019
|
+
"index", "idx_codex_files_alias_recent",
|
|
5020
|
+
"_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
|
|
5021
|
+
SchemaDeliveryObject(
|
|
5022
|
+
"index", "idx_codex_threads_recent",
|
|
5023
|
+
"_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
|
|
5024
|
+
SchemaDeliveryObject(
|
|
5025
|
+
"index", "idx_entries_model_time",
|
|
5026
|
+
"_apply_pricing_model_indexes", "047_spill_free_read_paths"),
|
|
5027
|
+
SchemaDeliveryObject(
|
|
5028
|
+
"index", "idx_codex_entries_model_time",
|
|
5029
|
+
"_apply_pricing_model_indexes", "047_spill_free_read_paths"),
|
|
5030
|
+
SchemaDeliveryObject(
|
|
5031
|
+
"index", "idx_codex_window_attributions_read_order",
|
|
5032
|
+
"_apply_codex_window_attribution_read_index",
|
|
5033
|
+
"047_spill_free_read_paths"),
|
|
4540
5034
|
)
|
|
4541
5035
|
|
|
4542
5036
|
|
|
@@ -5249,6 +5743,15 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5249
5743
|
_apply_codex_quota_unresolved_model_index(conn)
|
|
5250
5744
|
_apply_codex_quota_change_ledger(conn)
|
|
5251
5745
|
_apply_codex_accounting_change_ledger(conn)
|
|
5746
|
+
# #901 (spec §5.1-§5.3): the latest-capture summary, the general loader's
|
|
5747
|
+
# ordering index, the W2 lookup indexes, the pricing model indexes and the
|
|
5748
|
+
# window-attribution read order (Amendment 1 item 1). BEFORE the
|
|
5749
|
+
# legacy-FTS early-return, so an old-shape cache.db gains them.
|
|
5750
|
+
_apply_codex_quota_latest_summary(conn)
|
|
5751
|
+
_apply_codex_quota_load_order_index(conn)
|
|
5752
|
+
_apply_codex_metadata_lookup_indexes(conn)
|
|
5753
|
+
_apply_pricing_model_indexes(conn)
|
|
5754
|
+
_apply_codex_window_attribution_read_index(conn)
|
|
5252
5755
|
conn.execute(
|
|
5253
5756
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
5254
5757
|
"ON session_files(session_id)"
|
|
@@ -5394,6 +5897,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5394
5897
|
_apply_codex_find_projection_schema(conn)
|
|
5395
5898
|
_apply_conversation_generation_schema(conn)
|
|
5396
5899
|
_apply_codex_conversation_source_identity(conn)
|
|
5900
|
+
_apply_conversation_latest_meta_indexes(conn)
|
|
5397
5901
|
return
|
|
5398
5902
|
|
|
5399
5903
|
_apply_cache_schema(conn)
|
|
@@ -5425,6 +5929,10 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5425
5929
|
-- tests/test_conversations_transcripts_only.py, which derives its
|
|
5426
5930
|
-- expectation from COVERAGE_CACHE_FAMILIES rather than restating a list.
|
|
5427
5931
|
DROP TABLE IF EXISTS codex_window_attributions;
|
|
5932
|
+
-- #901: the latest-capture summary is a cache.db read model over
|
|
5933
|
+
-- quota_window_snapshots, which was dropped above together with its
|
|
5934
|
+
-- triggers and seek index. Drop the table as well.
|
|
5935
|
+
DROP TABLE IF EXISTS codex_quota_partition_latest;
|
|
5428
5936
|
|
|
5429
5937
|
CREATE TABLE IF NOT EXISTS conversation_source_files (
|
|
5430
5938
|
path TEXT PRIMARY KEY,
|
|
@@ -5478,6 +5986,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
5478
5986
|
_apply_codex_find_projection_schema(conn)
|
|
5479
5987
|
_apply_conversation_generation_schema(conn)
|
|
5480
5988
|
_apply_codex_conversation_source_identity(conn)
|
|
5989
|
+
_apply_conversation_latest_meta_indexes(conn)
|
|
5481
5990
|
|
|
5482
5991
|
|
|
5483
5992
|
def _ensure_codex_session_meta_provenance_index(
|
|
@@ -5654,6 +6163,36 @@ def _apply_codex_conversation_source_identity(conn: sqlite3.Connection) -> None:
|
|
|
5654
6163
|
conn, "codex_conversation_source_files", "inode", "INTEGER")
|
|
5655
6164
|
|
|
5656
6165
|
|
|
6166
|
+
def _apply_conversation_latest_meta_indexes(conn: sqlite3.Connection) -> None:
|
|
6167
|
+
"""The latest-metadata lookups' partial indexes (#901 Q11, 901-PA-001 b).
|
|
6168
|
+
|
|
6169
|
+
``_lib_conversation_query._session_latest_meta_map`` resolves a session's
|
|
6170
|
+
latest non-NULL ``cwd`` and latest non-NULL ``git_branch`` with two
|
|
6171
|
+
``LIMIT 1`` lookups ordered ``timestamp_utc DESC, id DESC``. Each partial
|
|
6172
|
+
index holds exactly the rows its lookup may return, in that order, so the
|
|
6173
|
+
lookup is one seek however long the session is and however sparse its
|
|
6174
|
+
metadata. The window query they replace sorted every touched session's
|
|
6175
|
+
whole history through temp b-trees on each conversation-sync pass (9.56 MB
|
|
6176
|
+
of temp in one measured call over a long session).
|
|
6177
|
+
|
|
6178
|
+
Called by the fresh-install schema apply and by conversations migration
|
|
6179
|
+
011, so both shapes converge. Skipped when ``conversation_messages`` is
|
|
6180
|
+
absent (a reachable state, not a corrupt one).
|
|
6181
|
+
"""
|
|
6182
|
+
if not _table_exists(conn, "conversation_messages"):
|
|
6183
|
+
return
|
|
6184
|
+
conn.execute(
|
|
6185
|
+
"CREATE INDEX IF NOT EXISTS idx_conv_session_latest_cwd "
|
|
6186
|
+
"ON conversation_messages(session_id, timestamp_utc, id) "
|
|
6187
|
+
"WHERE cwd IS NOT NULL"
|
|
6188
|
+
)
|
|
6189
|
+
conn.execute(
|
|
6190
|
+
"CREATE INDEX IF NOT EXISTS idx_conv_session_latest_git_branch "
|
|
6191
|
+
"ON conversation_messages(session_id, timestamp_utc, id) "
|
|
6192
|
+
"WHERE git_branch IS NOT NULL"
|
|
6193
|
+
)
|
|
6194
|
+
|
|
6195
|
+
|
|
5657
6196
|
def _apply_codex_find_projection_schema(conn: sqlite3.Connection) -> None:
|
|
5658
6197
|
"""Create #482's disposable visible-text projection tables.
|
|
5659
6198
|
|
|
@@ -5817,6 +6356,14 @@ CONVERSATIONS_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
5817
6356
|
"column", "codex_conversation_source_files.inode",
|
|
5818
6357
|
"_apply_codex_conversation_source_identity",
|
|
5819
6358
|
"010_codex_conversation_source_file_identity"),
|
|
6359
|
+
SchemaDeliveryObject(
|
|
6360
|
+
"index", "idx_conv_session_latest_cwd",
|
|
6361
|
+
"_apply_conversation_latest_meta_indexes",
|
|
6362
|
+
"011_conversation_latest_meta_indexes"),
|
|
6363
|
+
SchemaDeliveryObject(
|
|
6364
|
+
"index", "idx_conv_session_latest_git_branch",
|
|
6365
|
+
"_apply_conversation_latest_meta_indexes",
|
|
6366
|
+
"011_conversation_latest_meta_indexes"),
|
|
5820
6367
|
SchemaDeliveryObject(
|
|
5821
6368
|
"table", "claude_conversation_account_stamp_gaps",
|
|
5822
6369
|
"_apply_conversation_generation_schema",
|
|
@@ -6453,6 +7000,36 @@ def _conv_010_codex_conversation_source_file_identity(
|
|
|
6453
7000
|
conn.commit()
|
|
6454
7001
|
|
|
6455
7002
|
|
|
7003
|
+
@conversations_migration("011_conversation_latest_meta_indexes")
|
|
7004
|
+
def _conv_011_conversation_latest_meta_indexes(
|
|
7005
|
+
conn: sqlite3.Connection,
|
|
7006
|
+
) -> None:
|
|
7007
|
+
"""Deliver the latest-metadata partial indexes (#901 Q11, 901-PA-001 b).
|
|
7008
|
+
|
|
7009
|
+
``_session_latest_meta_map`` now resolves each touched session's latest
|
|
7010
|
+
``cwd`` and ``git_branch`` with two index-served ``LIMIT 1`` lookups; the
|
|
7011
|
+
indexes and the reasoning live on ``_apply_conversation_latest_meta_indexes``,
|
|
7012
|
+
the helper the fresh-install apply also calls. The schema apply that would
|
|
7013
|
+
create them is version-gated, so this registration is what reaches an
|
|
7014
|
+
already-current store.
|
|
7015
|
+
|
|
7016
|
+
Takes the Claude provider flock and defers cleanly on contention, like 009:
|
|
7017
|
+
building the indexes holds SQLite's write lock for a sort of the Claude
|
|
7018
|
+
transcript rows (one-time setup work), and a concurrent Claude transcript
|
|
7019
|
+
writer then reports contention instead of exhausting its busy timeout.
|
|
7020
|
+
|
|
7021
|
+
Re-running is a no-op (``IF NOT EXISTS``). NO self-stamp — the dispatcher
|
|
7022
|
+
central-stamps on a clean return (#140).
|
|
7023
|
+
"""
|
|
7024
|
+
held = _acquire_conversations_db_claude_provider_flock(
|
|
7025
|
+
conn, migration="conversations 011 latest metadata indexes")
|
|
7026
|
+
try:
|
|
7027
|
+
_apply_conversation_latest_meta_indexes(conn)
|
|
7028
|
+
conn.commit()
|
|
7029
|
+
finally:
|
|
7030
|
+
_release_cache_db_writer_flocks(held)
|
|
7031
|
+
|
|
7032
|
+
|
|
6456
7033
|
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
6457
7034
|
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
6458
7035
|
# pair. The three column names MUST match the conversation_messages columns BY
|
|
@@ -9277,6 +9854,70 @@ def _046_codex_source_file_identity(conn: sqlite3.Connection) -> None:
|
|
|
9277
9854
|
conn.commit()
|
|
9278
9855
|
|
|
9279
9856
|
|
|
9857
|
+
@cache_migration("047_spill_free_read_paths")
|
|
9858
|
+
def _047_spill_free_read_paths(conn: sqlite3.Connection) -> None:
|
|
9859
|
+
"""#901 (spec §5.1-§5.3): deliver the spill-free read paths to existing stores.
|
|
9860
|
+
|
|
9861
|
+
``_apply_cache_schema`` creates every object below for a new store, but that
|
|
9862
|
+
pass is VERSION-GATED: ``open_cache_db`` runs it only when ``user_version``
|
|
9863
|
+
differs from ``len(_CACHE_MIGRATIONS)``. Registering here bumps the head,
|
|
9864
|
+
the mechanism migrations 029, 031, 044, 045 and 046 rely on.
|
|
9865
|
+
|
|
9866
|
+
The handler ALWAYS re-derives the latest-capture summary from the physical
|
|
9867
|
+
rows (``rebootstrap=True``), so a re-upgrade after an older binary trimmed
|
|
9868
|
+
this marker converges on the rows actually stored whatever happened to the
|
|
9869
|
+
summary meanwhile. On a first upgrade the schema apply has already
|
|
9870
|
+
bootstrapped it once; the second derivation is one ordered scan of the new
|
|
9871
|
+
seek index, paid once.
|
|
9872
|
+
|
|
9873
|
+
It also delivers ``idx_codex_window_attributions_read_order`` (Amendment 1
|
|
9874
|
+
item 1, ``c901-design-2``), the window-attribution read's assertion order.
|
|
9875
|
+
|
|
9876
|
+
Takes the Codex provider flock like handlers 024-027, 034 and 043, because
|
|
9877
|
+
the summary is Codex-derived, and DEFERS (``MigrationGateNotMet``) before
|
|
9878
|
+
any DDL when a Codex sync holds it. Idempotent: every object uses IF NOT
|
|
9879
|
+
EXISTS or drop-then-create, and a re-derivation over its own output writes
|
|
9880
|
+
the same rows. NO self-stamp — the dispatcher central-stamps on a clean
|
|
9881
|
+
return (#140).
|
|
9882
|
+
"""
|
|
9883
|
+
held = _acquire_cache_db_codex_provider_flock(
|
|
9884
|
+
conn, migration="047 spill-free read paths")
|
|
9885
|
+
try:
|
|
9886
|
+
_apply_codex_quota_latest_summary(conn, rebootstrap=True)
|
|
9887
|
+
_apply_codex_quota_load_order_index(conn)
|
|
9888
|
+
_apply_codex_metadata_lookup_indexes(conn)
|
|
9889
|
+
_apply_pricing_model_indexes(conn)
|
|
9890
|
+
_apply_codex_window_attribution_read_index(conn)
|
|
9891
|
+
conn.commit()
|
|
9892
|
+
finally:
|
|
9893
|
+
_release_cache_db_writer_flocks(held)
|
|
9894
|
+
|
|
9895
|
+
|
|
9896
|
+
@cache_migration("048_codex_quota_physical_group_order")
|
|
9897
|
+
def _048_codex_quota_physical_group_order(conn: sqlite3.Connection) -> None:
|
|
9898
|
+
"""#901 Q11 (spec §5.3a, 901-PA-001 a): stream every physical-group shard.
|
|
9899
|
+
|
|
9900
|
+
Replaces ``idx_qws_physical_group``'s five-member definition with the same
|
|
9901
|
+
name, partial predicate and five equality members followed by the loader's
|
|
9902
|
+
order columns ``captured_at_utc, resets_at_utc, source_path,
|
|
9903
|
+
line_offset``. The shard SQL, its population and its order are unchanged;
|
|
9904
|
+
only the sorter goes. The reasoning lives on
|
|
9905
|
+
``_apply_codex_quota_group_index``, which this delegates to so the
|
|
9906
|
+
migration and the schema apply cannot drift, and which replaces the index
|
|
9907
|
+
atomically.
|
|
9908
|
+
|
|
9909
|
+
Same version-gate reason as 040: the schema apply that converges the index
|
|
9910
|
+
is skipped by a steady-state open, so registering here is what makes an
|
|
9911
|
+
already-current install pick it up. The rebuild is a one-time sort of the
|
|
9912
|
+
Codex quota rows (setup work, not a recurring path).
|
|
9913
|
+
|
|
9914
|
+
Re-running is a no-op: a current index is left alone. NO self-stamp — the
|
|
9915
|
+
dispatcher central-stamps on a clean return (#140).
|
|
9916
|
+
"""
|
|
9917
|
+
_apply_codex_quota_group_index(conn)
|
|
9918
|
+
conn.commit()
|
|
9919
|
+
|
|
9920
|
+
|
|
9280
9921
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
9281
9922
|
|
|
9282
9923
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -10867,6 +11508,13 @@ def _repair_marker_path(path: pathlib.Path) -> pathlib.Path:
|
|
|
10867
11508
|
return path.with_name(f"{path.name}.repairing")
|
|
10868
11509
|
|
|
10869
11510
|
|
|
11511
|
+
def _keeper_yield_request_path(path: pathlib.Path) -> pathlib.Path:
|
|
11512
|
+
"""#901 W9 (Q15): the store-scoped request that makes every process's
|
|
11513
|
+
idle checkpoint keeper (`_lib_wal_checkpoint`) close. Its owner record is
|
|
11514
|
+
the repair marker's, written, read and checked for liveness the same way."""
|
|
11515
|
+
return path.with_name(f"{path.name}.keeper-yield")
|
|
11516
|
+
|
|
11517
|
+
|
|
10870
11518
|
_REPAIR_OWNER_VERSION = 1
|
|
10871
11519
|
_REPAIR_OWNER_UNKNOWN_IDENTITY_LEASE_SECONDS = 30 * 60
|
|
10872
11520
|
|
|
@@ -11019,10 +11667,30 @@ def _claim_repair_marker(
|
|
|
11019
11667
|
path: pathlib.Path,
|
|
11020
11668
|
) -> "tuple[RepairMarkerClaim | None, str]":
|
|
11021
11669
|
"""Atomically publish a complete owner record; reclaim stale ownership."""
|
|
11022
|
-
|
|
11670
|
+
return _claim_owner_record(
|
|
11671
|
+
_repair_marker_path(path),
|
|
11672
|
+
lambda: _remove_stale_repair_marker(path),
|
|
11673
|
+
owner="repair-owner",
|
|
11674
|
+
record="repair marker",
|
|
11675
|
+
)
|
|
11676
|
+
|
|
11677
|
+
|
|
11678
|
+
def _claim_owner_record(
|
|
11679
|
+
marker: pathlib.Path,
|
|
11680
|
+
remove_stale: "Callable[[], tuple[bool, str]]",
|
|
11681
|
+
*,
|
|
11682
|
+
owner: str,
|
|
11683
|
+
record: str,
|
|
11684
|
+
) -> "tuple[RepairMarkerClaim | None, str]":
|
|
11685
|
+
"""Publish ``marker`` with a complete owner record by exclusive create.
|
|
11686
|
+
|
|
11687
|
+
The repair marker's discipline, shared with #901's keeper-yield request:
|
|
11688
|
+
the record is written and fsynced under a private name and linked into
|
|
11689
|
+
place, so it is complete or absent; an existing record is replaced only
|
|
11690
|
+
when ``remove_stale`` proves its owner dead or its pid reused."""
|
|
11023
11691
|
process_start = _process_start_identity(os.getpid())
|
|
11024
11692
|
if process_start is None:
|
|
11025
|
-
raise OSError("could not establish
|
|
11693
|
+
raise OSError(f"could not establish {owner} process identity")
|
|
11026
11694
|
for _attempt in range(3):
|
|
11027
11695
|
claim_id = f"{os.getpid()}-{time.time_ns()}-{os.urandom(8).hex()}"
|
|
11028
11696
|
temp = marker.with_name(f".{marker.name}.{claim_id}.tmp")
|
|
@@ -11041,7 +11709,7 @@ def _claim_repair_marker(
|
|
|
11041
11709
|
try:
|
|
11042
11710
|
os.link(temp, marker)
|
|
11043
11711
|
except FileExistsError:
|
|
11044
|
-
removed, reason =
|
|
11712
|
+
removed, reason = remove_stale()
|
|
11045
11713
|
if not removed:
|
|
11046
11714
|
return None, reason
|
|
11047
11715
|
continue
|
|
@@ -11052,7 +11720,7 @@ def _claim_repair_marker(
|
|
|
11052
11720
|
temp.unlink()
|
|
11053
11721
|
except FileNotFoundError:
|
|
11054
11722
|
pass
|
|
11055
|
-
return None, f"could not claim
|
|
11723
|
+
return None, f"could not claim {record} {marker}"
|
|
11056
11724
|
|
|
11057
11725
|
|
|
11058
11726
|
def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None:
|
|
@@ -11067,8 +11735,135 @@ def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None
|
|
|
11067
11735
|
_fsync_directory(marker.parent)
|
|
11068
11736
|
|
|
11069
11737
|
|
|
11738
|
+
#: #901 W9 (Q15): the one monotonic deadline a process that needs a
|
|
11739
|
+
#: cache.db or conversations.db family exclusive or drained waits for the
|
|
11740
|
+
#: other processes' idle checkpoint keepers to close, per store: twice the
|
|
11741
|
+
#: keepers' 10 s timer interval. Never renewed per keeper, scan or retry.
|
|
11742
|
+
_KEEPER_YIELD_DEADLINE_SECONDS = 20.0
|
|
11743
|
+
#: The pause between two handle checks, or two exclusive acquisitions, while
|
|
11744
|
+
#: a requester waits under that deadline.
|
|
11745
|
+
_KEEPER_YIELD_RETRY_SECONDS = 0.25
|
|
11746
|
+
|
|
11747
|
+
|
|
11748
|
+
def _remove_stale_keeper_yield_request(
|
|
11749
|
+
path: pathlib.Path,
|
|
11750
|
+
) -> "tuple[bool, str]":
|
|
11751
|
+
"""Remove a provably stale keeper-yield request (dead or reused-pid
|
|
11752
|
+
owner), as `_remove_stale_repair_marker` does for the repair marker."""
|
|
11753
|
+
request = _keeper_yield_request_path(path)
|
|
11754
|
+
if not request.exists():
|
|
11755
|
+
return True, ""
|
|
11756
|
+
live, reason = _repair_marker_is_live(request)
|
|
11757
|
+
if live:
|
|
11758
|
+
return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
|
|
11759
|
+
try:
|
|
11760
|
+
request.unlink()
|
|
11761
|
+
except FileNotFoundError:
|
|
11762
|
+
return True, ""
|
|
11763
|
+
except OSError as exc:
|
|
11764
|
+
return False, f"could not remove stale keeper-yield request {request}: {exc}"
|
|
11765
|
+
_fsync_directory(request.parent)
|
|
11766
|
+
return True, reason
|
|
11767
|
+
|
|
11768
|
+
|
|
11769
|
+
def _claim_keeper_yield_request(
|
|
11770
|
+
path: pathlib.Path,
|
|
11771
|
+
) -> "tuple[RepairMarkerClaim | None, str]":
|
|
11772
|
+
"""Publish ``<store>.keeper-yield`` with this process as its owner."""
|
|
11773
|
+
return _claim_owner_record(
|
|
11774
|
+
_keeper_yield_request_path(path),
|
|
11775
|
+
lambda: _remove_stale_keeper_yield_request(path),
|
|
11776
|
+
owner="keeper-yield-owner",
|
|
11777
|
+
record="keeper-yield request",
|
|
11778
|
+
)
|
|
11779
|
+
|
|
11780
|
+
|
|
11781
|
+
def _release_keeper_yield_request(
|
|
11782
|
+
path: pathlib.Path, claim: RepairMarkerClaim,
|
|
11783
|
+
) -> None:
|
|
11784
|
+
"""Remove the request; only its owner does, and a record someone else now
|
|
11785
|
+
owns is left alone. Best effort: a request left behind fences keepers
|
|
11786
|
+
only while this process lives, and it is stale once the process exits."""
|
|
11787
|
+
request = _keeper_yield_request_path(path)
|
|
11788
|
+
owner = _read_repair_owner(request)
|
|
11789
|
+
if owner is not None and owner.get("claim_id") != claim.claim_id:
|
|
11790
|
+
return
|
|
11791
|
+
try:
|
|
11792
|
+
request.unlink()
|
|
11793
|
+
except OSError:
|
|
11794
|
+
return
|
|
11795
|
+
try:
|
|
11796
|
+
_fsync_directory(request.parent)
|
|
11797
|
+
except OSError:
|
|
11798
|
+
pass
|
|
11799
|
+
|
|
11800
|
+
|
|
11801
|
+
@contextlib.contextmanager
|
|
11802
|
+
def _keeper_yield_request(path) -> "Iterator[float]":
|
|
11803
|
+
"""Ask every process's idle checkpoint keeper on the store to close (#901
|
|
11804
|
+
W9, Q15) and yield the one monotonic deadline to wait for them.
|
|
11805
|
+
|
|
11806
|
+
The caller already holds its maintenance and provider locks, taken as
|
|
11807
|
+
before and failing as promptly as before; the request grants no authority
|
|
11808
|
+
of its own. It is published with the repair marker's owner record, this
|
|
11809
|
+
process's own keeper is closed at once, and the request is removed when
|
|
11810
|
+
the caller finishes or fails. The other processes observe it at their next
|
|
11811
|
+
`arm`, timer round or exit and close their keepers without a checkpoint.
|
|
11812
|
+
When it cannot be published the caller still waits, because a sync or a
|
|
11813
|
+
reader may close in time; it then refuses at the deadline as before."""
|
|
11814
|
+
path = pathlib.Path(path)
|
|
11815
|
+
try:
|
|
11816
|
+
claim, _reason = _claim_keeper_yield_request(path)
|
|
11817
|
+
except OSError:
|
|
11818
|
+
claim = None
|
|
11819
|
+
try:
|
|
11820
|
+
try:
|
|
11821
|
+
import _lib_wal_checkpoint
|
|
11822
|
+
|
|
11823
|
+
_lib_wal_checkpoint.release(path)
|
|
11824
|
+
except Exception: # noqa: BLE001 — the drain check below decides
|
|
11825
|
+
pass
|
|
11826
|
+
yield time.monotonic() + _KEEPER_YIELD_DEADLINE_SECONDS
|
|
11827
|
+
finally:
|
|
11828
|
+
if claim is not None:
|
|
11829
|
+
_release_keeper_yield_request(path, claim)
|
|
11830
|
+
|
|
11831
|
+
|
|
11832
|
+
def _await_family_drained(
|
|
11833
|
+
path: pathlib.Path, deadline: float,
|
|
11834
|
+
) -> "set[int] | None":
|
|
11835
|
+
"""Repeat the family's handle check until it finds no open handle or the
|
|
11836
|
+
deadline passes (#901 W9, Q15); return the last answer.
|
|
11837
|
+
|
|
11838
|
+
``None`` (unknown) returns at once, as a refusal. Nothing is killed,
|
|
11839
|
+
ignored or exempted: at the deadline the caller refuses with today's
|
|
11840
|
+
message for whatever handle remains — a keeper that did not yield, a sync
|
|
11841
|
+
in progress, a reader."""
|
|
11842
|
+
while True:
|
|
11843
|
+
open_pids = _db_family_open_pids(path)
|
|
11844
|
+
if not open_pids:
|
|
11845
|
+
return open_pids
|
|
11846
|
+
remaining = deadline - time.monotonic()
|
|
11847
|
+
if remaining <= 0:
|
|
11848
|
+
return open_pids
|
|
11849
|
+
time.sleep(min(_KEEPER_YIELD_RETRY_SECONDS, remaining))
|
|
11850
|
+
|
|
11851
|
+
|
|
11070
11852
|
def _db_family_open_pids(path: pathlib.Path) -> "set[int] | None":
|
|
11071
|
-
"""Return processes with any SQLite family member open; None if unknown.
|
|
11853
|
+
"""Return processes with any SQLite family member open; None if unknown.
|
|
11854
|
+
|
|
11855
|
+
#901 W9 (Q14): this process's idle keeper for the store is released first,
|
|
11856
|
+
without a checkpoint. It exists only to defer checkpoints, and every
|
|
11857
|
+
caller asks this question before replacing or quarantining the family, so
|
|
11858
|
+
the keeper must never be the handle that refuses this process's own store
|
|
11859
|
+
recovery; the next sync's arm reopens it on whatever file is then there.
|
|
11860
|
+
"""
|
|
11861
|
+
try:
|
|
11862
|
+
import _lib_wal_checkpoint
|
|
11863
|
+
|
|
11864
|
+
_lib_wal_checkpoint.release(path)
|
|
11865
|
+
except Exception: # noqa: BLE001 — the drain check below still decides
|
|
11866
|
+
pass
|
|
11072
11867
|
family = [
|
|
11073
11868
|
pathlib.Path(str(path) + suffix)
|
|
11074
11869
|
for suffix in ("", "-journal", "-wal", "-shm")
|
|
@@ -12065,13 +12860,51 @@ def _vacuum_required_free_bytes(path) -> int:
|
|
|
12065
12860
|
return 2 * db_bytes + wal_bytes
|
|
12066
12861
|
|
|
12067
12862
|
|
|
12068
|
-
def
|
|
12863
|
+
def _acquire_vacuum_exclusion(
|
|
12864
|
+
conn: sqlite3.Connection, deadline: "float | None",
|
|
12865
|
+
) -> None:
|
|
12866
|
+
"""Take a WAL store's EXCLUSIVE lock and keep it (#901 W9, Q15).
|
|
12867
|
+
|
|
12868
|
+
Under ``locking_mode=EXCLUSIVE`` a write transaction takes the database
|
|
12869
|
+
file's EXCLUSIVE lock and the connection keeps it after the transaction
|
|
12870
|
+
ends, so ``BEGIN IMMEDIATE`` then ``ROLLBACK`` acquires the exclusion
|
|
12871
|
+
without writing anything, and the checkpoint and VACUUM that follow run
|
|
12872
|
+
under one continuous hold (#386). Only this acquisition is retried — each
|
|
12873
|
+
attempt under the existing busy timeout, a short pause between attempts —
|
|
12874
|
+
until ``deadline``, the keepers' drain grace; the VACUUM is never retried.
|
|
12875
|
+
``deadline=None`` makes one attempt. The last busy error is raised at the
|
|
12876
|
+
deadline, so the caller's refusal is today's."""
|
|
12877
|
+
while True:
|
|
12878
|
+
try:
|
|
12879
|
+
conn.execute("BEGIN IMMEDIATE")
|
|
12880
|
+
except sqlite3.OperationalError as exc:
|
|
12881
|
+
text = str(exc).lower()
|
|
12882
|
+
if (
|
|
12883
|
+
deadline is None
|
|
12884
|
+
or ("lock" not in text and "busy" not in text)
|
|
12885
|
+
or time.monotonic() >= deadline
|
|
12886
|
+
):
|
|
12887
|
+
raise
|
|
12888
|
+
time.sleep(max(0.0, min(_KEEPER_YIELD_RETRY_SECONDS,
|
|
12889
|
+
deadline - time.monotonic())))
|
|
12890
|
+
continue
|
|
12891
|
+
conn.execute("ROLLBACK")
|
|
12892
|
+
return
|
|
12893
|
+
|
|
12894
|
+
|
|
12895
|
+
def _run_vacuum_exclusive(
|
|
12896
|
+
path, label: str, *, deadline: "float | None" = None,
|
|
12897
|
+
) -> int:
|
|
12069
12898
|
"""VACUUM ``path`` under EXCLUSIVE; checkpoint WAL-backed stores first.
|
|
12070
12899
|
|
|
12071
12900
|
``locking_mode=EXCLUSIVE`` + a short ``busy_timeout`` make a concurrent
|
|
12072
|
-
reader/writer
|
|
12073
|
-
lock, which the advisory flocks do not provide against dashboard
|
|
12074
|
-
|
|
12901
|
+
reader/writer fail rather than hang (no TOCTOU gap — the exclusion is the
|
|
12902
|
+
DB's own lock, which the advisory flocks do not provide against dashboard
|
|
12903
|
+
readers). On a WAL store (#901 W9, Q15) the exclusive acquisition comes
|
|
12904
|
+
first and is retried until ``deadline`` while other processes' idle
|
|
12905
|
+
checkpoint keepers yield to the caller's request; the checkpoint and the
|
|
12906
|
+
VACUUM then run under that one hold. Exit 0 on success, 3 when the DB is
|
|
12907
|
+
in use."""
|
|
12075
12908
|
conn = sqlite3.connect(f"file:{path}?mode=rw", uri=True)
|
|
12076
12909
|
try:
|
|
12077
12910
|
conn.execute(f"PRAGMA busy_timeout={_VACUUM_BUSY_TIMEOUT_MS}")
|
|
@@ -12080,6 +12913,7 @@ def _run_vacuum_exclusive(path, label: str) -> int:
|
|
|
12080
12913
|
try:
|
|
12081
12914
|
conn.execute("PRAGMA locking_mode=EXCLUSIVE")
|
|
12082
12915
|
if label != "stats.db":
|
|
12916
|
+
_acquire_vacuum_exclusion(conn, deadline)
|
|
12083
12917
|
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
12084
12918
|
conn.execute("VACUUM")
|
|
12085
12919
|
except sqlite3.OperationalError as exc:
|
|
@@ -12189,7 +13023,13 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
|
|
|
12189
13023
|
"before retrying VACUUM."
|
|
12190
13024
|
)
|
|
12191
13025
|
return 3
|
|
12192
|
-
|
|
13026
|
+
# stats.db has no checkpoint keeper: one attempt, as before.
|
|
13027
|
+
return _run_vacuum_exclusive(path, label)
|
|
13028
|
+
# #901 W9 (Q15): with both locks held, ask the other processes'
|
|
13029
|
+
# idle checkpoint keepers to close, close this process's own, and
|
|
13030
|
+
# retry only the exclusive acquisition within the drain grace.
|
|
13031
|
+
with _keeper_yield_request(path) as deadline:
|
|
13032
|
+
return _run_vacuum_exclusive(path, label, deadline=deadline)
|
|
12193
13033
|
finally:
|
|
12194
13034
|
for fh in held:
|
|
12195
13035
|
try:
|
|
@@ -12212,9 +13052,12 @@ def cmd_db_vacuum(args: argparse.Namespace) -> int:
|
|
|
12212
13052
|
|
|
12213
13053
|
NEVER automatic. Holds the maintenance flock + (for cache.db) the provider
|
|
12214
13054
|
flocks, then runs a checkpoint + VACUUM under a real SQLite EXCLUSIVE lock so
|
|
12215
|
-
a concurrent dashboard reader
|
|
12216
|
-
|
|
12217
|
-
|
|
13055
|
+
a concurrent dashboard reader makes it refuse instead of racing. For
|
|
13056
|
+
cache.db and conversations.db it first asks other processes' idle
|
|
13057
|
+
checkpoint keepers to close (#901 W9, Q15) and retries the exclusive
|
|
13058
|
+
acquisition for at most 20 s; a handle still open then gets the same
|
|
13059
|
+
refusal. Refuses when free disk is below ~2x the file + WAL. Exit 0 on
|
|
13060
|
+
success, 3 when a target is in use or disk is short."""
|
|
12218
13061
|
which = getattr(args, "db", "cache")
|
|
12219
13062
|
targets = []
|
|
12220
13063
|
if which in ("cache", "all"):
|