cctally 1.112.0 → 1.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/README.md +4 -5
  3. package/bin/_cctally_cache.py +1683 -485
  4. package/bin/_cctally_core.py +223 -1
  5. package/bin/_cctally_dashboard.py +96 -18
  6. package/bin/_cctally_dashboard_perf.py +128 -9
  7. package/bin/_cctally_dashboard_sources.py +267 -204
  8. package/bin/_cctally_db.py +861 -18
  9. package/bin/_cctally_diagnosis_sources.py +22 -4
  10. package/bin/_cctally_doctor.py +136 -18
  11. package/bin/_cctally_five_hour.py +2 -2
  12. package/bin/_cctally_forecast.py +27 -6
  13. package/bin/_cctally_journal.py +67 -5
  14. package/bin/_cctally_milestone_history.py +82 -9
  15. package/bin/_cctally_percent_breakdown.py +3 -2
  16. package/bin/_cctally_quota.py +201 -33
  17. package/bin/_cctally_record.py +14 -0
  18. package/bin/_cctally_rederive.py +23 -9
  19. package/bin/_cctally_source_analytics.py +60 -15
  20. package/bin/_cctally_statusline.py +17 -0
  21. package/bin/_cctally_store.py +80 -5
  22. package/bin/_cctally_tui.py +4 -3
  23. package/bin/_cctally_weekrefs.py +128 -21
  24. package/bin/_lib_cache_report.py +35 -6
  25. package/bin/_lib_codex_conversation.py +32 -0
  26. package/bin/_lib_codex_conversation_query.py +164 -30
  27. package/bin/_lib_conversation_query.py +95 -56
  28. package/bin/_lib_conversation_retention.py +1605 -640
  29. package/bin/_lib_cost_provenance.py +53 -2
  30. package/bin/_lib_doctor.py +263 -69
  31. package/bin/_lib_fmt.py +11 -0
  32. package/bin/_lib_merge_gate.py +469 -51
  33. package/bin/_lib_perf.py +27 -1
  34. package/bin/_lib_pricing.py +209 -8
  35. package/bin/_lib_pricing_check.py +20 -4
  36. package/bin/_lib_reclaim_planner.py +792 -0
  37. package/bin/_lib_rederive.py +225 -77
  38. package/bin/_lib_snapshot_cache.py +21 -0
  39. package/bin/_lib_sqlite_close.py +81 -0
  40. package/bin/_lib_subscription_weeks.py +100 -17
  41. package/bin/_lib_tick_stats.py +217 -17
  42. package/bin/_lib_wal_checkpoint.py +1077 -0
  43. package/bin/_lib_write_budget.py +483 -0
  44. package/bin/_lib_write_io.py +703 -0
  45. package/bin/cctally +8 -0
  46. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  47. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  48. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  49. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  50. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  51. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  52. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  53. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  54. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  55. package/dashboard/static/dashboard.html +2 -2
  56. package/package.json +6 -1
  57. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  58. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -60,6 +60,7 @@ Spec: docs/superpowers/specs/2026-05-13-bin-cctally-split-design.md
60
60
  from __future__ import annotations
61
61
 
62
62
  import argparse
63
+ import contextlib
63
64
  import contextvars
64
65
  import datetime as dt
65
66
  import enum
@@ -76,7 +77,7 @@ import tempfile
76
77
  import time
77
78
  import traceback
78
79
  from dataclasses import dataclass
79
- from typing import Any, Callable, NamedTuple
80
+ from typing import Any, Callable, Iterator, NamedTuple
80
81
 
81
82
 
82
83
  def _cctally():
@@ -3876,13 +3877,29 @@ def _codex_quota_ledger_ddl() -> tuple[str, ...]:
3876
3877
  #: because SQLite only uses an expression index when the query's expression
3877
3878
  #: matches the indexed one — indexing the bare ``COALESCE`` while the reader
3878
3879
  #: wraps it in ``unixepoch`` silently buys nothing.
3880
+ #:
3881
+ #: #901 Q11 (spec §5.3a, 901-PA-001 a; cache migration 048): the four order
3882
+ #: columns after the five equality members are the loader's ``ORDER BY
3883
+ #: source_root_key, captured_at_utc, resets_at_utc, source_path, line_offset``
3884
+ #: minus its constant first term, so each physical-group shard streams from the
3885
+ #: seek. Without them every shard re-sorted its group's whole history ("USE TEMP
3886
+ #: B-TREE FOR LAST 4 TERMS OF ORDER BY"; 3.3, 12.5 and 5.4 MB of temp in three
3887
+ #: measured calls), and one window group is a week of activity, not a bound.
3879
3888
  _QUOTA_GROUP_INDEX_DDL = (
3880
3889
  "CREATE INDEX IF NOT EXISTS idx_qws_physical_group "
3881
3890
  "ON quota_window_snapshots("
3882
3891
  " source_root_key, logical_limit_key, observed_slot, window_minutes,"
3883
- " unixepoch(COALESCE(canonical_resets_at_utc, resets_at_utc)))"
3892
+ " unixepoch(COALESCE(canonical_resets_at_utc, resets_at_utc)),"
3893
+ " captured_at_utc, resets_at_utc, source_path, line_offset)"
3884
3894
  " WHERE source='codex'"
3885
3895
  )
3896
+ #: The key columns ``PRAGMA index_xinfo`` reports for the current definition
3897
+ #: (``None`` is the expression member). An index of the same name with any
3898
+ #: other key list is a pre-048 definition and is replaced.
3899
+ _QUOTA_GROUP_INDEX_KEY_COLUMNS = (
3900
+ "source_root_key", "logical_limit_key", "observed_slot", "window_minutes",
3901
+ None, "captured_at_utc", "resets_at_utc", "source_path", "line_offset",
3902
+ )
3886
3903
 
3887
3904
 
3888
3905
  def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
@@ -3916,6 +3933,16 @@ def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
3916
3933
  Guarded on ``canonical_resets_at_utc`` for the same reason the ledger is: a
3917
3934
  legacy-shape cache whose schema apply took the FTS early-return before that
3918
3935
  column add would raise ``no such column`` here.
3936
+
3937
+ #901 Q11 (cache migration 048) extends the key list with the loader's order
3938
+ columns (see ``_QUOTA_GROUP_INDEX_DDL``). ``CREATE INDEX IF NOT EXISTS``
3939
+ never reshapes an existing index, so an index of this name with any other
3940
+ key list is dropped and recreated, and the two statements commit together
3941
+ (its own ``BEGIN IMMEDIATE``, or a savepoint inside a caller's
3942
+ transaction): the loader probes for the index by name and then pins it with
3943
+ ``INDEXED BY``, so a reader must never find it missing in between. A
3944
+ current index is left alone, so the helper is idempotent and costs one
3945
+ ``PRAGMA`` on a current store.
3919
3946
  """
3920
3947
  cols = {
3921
3948
  str(row[1]) for row in conn.execute(
@@ -3923,8 +3950,34 @@ def _apply_codex_quota_group_index(conn: sqlite3.Connection) -> None:
3923
3950
  }
3924
3951
  if not cols or "canonical_resets_at_utc" not in cols:
3925
3952
  return
3926
- conn.execute(_QUOTA_GROUP_INDEX_DDL)
3927
- conn.execute("DROP INDEX IF EXISTS idx_qws_window_ident")
3953
+ current = tuple(
3954
+ row[2] for row in conn.execute(
3955
+ "PRAGMA index_xinfo(idx_qws_physical_group)") if row[5])
3956
+ retired = conn.execute(
3957
+ "SELECT 1 FROM sqlite_master WHERE type='index'"
3958
+ " AND name='idx_qws_window_ident'").fetchone() is not None
3959
+ if current == _QUOTA_GROUP_INDEX_KEY_COLUMNS and not retired:
3960
+ return
3961
+ own_transaction = not conn.in_transaction
3962
+ conn.execute(
3963
+ "BEGIN IMMEDIATE" if own_transaction
3964
+ else "SAVEPOINT codex_quota_group_index")
3965
+ try:
3966
+ if current and current != _QUOTA_GROUP_INDEX_KEY_COLUMNS:
3967
+ conn.execute("DROP INDEX IF EXISTS idx_qws_physical_group")
3968
+ conn.execute(_QUOTA_GROUP_INDEX_DDL)
3969
+ conn.execute("DROP INDEX IF EXISTS idx_qws_window_ident")
3970
+ except BaseException:
3971
+ if own_transaction:
3972
+ conn.rollback()
3973
+ else:
3974
+ conn.execute("ROLLBACK TO codex_quota_group_index")
3975
+ conn.execute("RELEASE codex_quota_group_index")
3976
+ raise
3977
+ if own_transaction:
3978
+ conn.commit()
3979
+ else:
3980
+ conn.execute("RELEASE codex_quota_group_index")
3928
3981
 
3929
3982
 
3930
3983
  #: Partial index over the rows the standing `observed_model` resolution can
@@ -4283,6 +4336,410 @@ def _apply_codex_accounting_change_ledger(conn: sqlite3.Connection) -> None:
4283
4336
  conn.execute(statement)
4284
4337
 
4285
4338
 
4339
+ # ── #901 W1: the maintained raw-partition maxima (spec §5.1, T1) ─────────────
4340
+ #
4341
+ # `load_codex_quota_observations(latest_per_identity=True)` computed
4342
+ # `MAX(unixepoch(captured_at_utc)) OVER (PARTITION BY <nine raw columns>)` across
4343
+ # every retained Codex quota row on every doctor refresh; on the measured store
4344
+ # that sorted 565K rows through ~1.1 GB of temp files per call (spec §1.2). The
4345
+ # table below holds exactly that window's per-partition maximum, maintained by
4346
+ # triggers in the same transaction as every mutation, so the latest read becomes
4347
+ # a scan of ~10^3 summary rows plus one index seek per partition for the rows
4348
+ # tied at its maximum second (tests/test_901_partition_latest_maintenance.py
4349
+ # holds it to the frozen window query after every writer class).
4350
+ #
4351
+ # Keys are `quote(<raw column>)`: NOT NULL, collision-free across NULL, '' and
4352
+ # storage classes, and equal exactly when the window's PARTITION BY puts two
4353
+ # rows in one partition (no collation on these columns; TEXT affinity stores
4354
+ # numbers as text and INTEGER affinity integral reals as integers, so value
4355
+ # equality and quote() equality coincide). The key columns are declared BLOB on
4356
+ # purpose: a TEXT key column gives `quote(q.col) = s.key` TEXT comparison
4357
+ # affinity, and SQLite then refuses the expression index (measured: the tie
4358
+ # fetch fell back to walking the whole `source` autoindex). The seek index leads
4359
+ # with `source` for the same measured reason.
4360
+ #
4361
+ # The triggers use built-in functions only (quote, trim, coalesce, unixepoch),
4362
+ # so an older binary that opens the store keeps them firing. They never upsert:
4363
+ # an outer statement's conflict policy propagates into trigger-body statements,
4364
+ # so every repair is a DELETE followed by an INSERT that cannot conflict. A
4365
+ # REPLACE against the snapshots table would delete rows without firing the
4366
+ # delete trigger (recursive_triggers is off); no writer issues one, and
4367
+ # test_no_writer_replaces_quota_rows fails if one appears.
4368
+ #
4369
+ # `unixepoch()` in an index expression throws on the literal 'now', so the seek
4370
+ # index refuses a Codex row whose capture is 'now' — the same property
4371
+ # idx_qws_physical_group already has for the reset. Ingest writes only
4372
+ # `_utc_iso` instants.
4373
+
4374
+ #: Must equal `_cctally_quota._CODEX_QUOTA_REQUIRED_TEXT` (#500: the loader and
4375
+ #: every population derived for it agree on one required-text set);
4376
+ #: tests/test_901_partition_latest_maintenance.py pins the equality.
4377
+ _CODEX_QUOTA_LATEST_REQUIRED_TEXT = (
4378
+ "source", "source_root_key", "source_path", "captured_at_utc",
4379
+ "observed_slot", "logical_limit_key", "resets_at_utc",
4380
+ )
4381
+
4382
+ #: (summary key column, raw column; None means the coalesced reset anchor), in
4383
+ #: the window query's PARTITION BY order.
4384
+ _CODEX_QUOTA_LATEST_KEYS = (
4385
+ ("k_source_root_key", "source_root_key"),
4386
+ ("k_logical_limit_key", "logical_limit_key"),
4387
+ ("k_observed_slot", "observed_slot"),
4388
+ ("k_window_minutes", "window_minutes"),
4389
+ ("k_limit_id", "limit_id"),
4390
+ ("k_limit_name", "limit_name"),
4391
+ ("k_observed_model", "observed_model"),
4392
+ ("k_account_key", "account_key"),
4393
+ ("k_reset_anchor", None),
4394
+ )
4395
+ _CODEX_QUOTA_LATEST_KEY_COLUMNS = tuple(
4396
+ key for key, _column in _CODEX_QUOTA_LATEST_KEYS)
4397
+
4398
+ #: Every column a partition key or a validity predicate reads.
4399
+ _CODEX_QUOTA_LATEST_UPDATE_OF = (
4400
+ "source", "source_root_key", "source_path", "captured_at_utc",
4401
+ "observed_slot", "logical_limit_key", "resets_at_utc",
4402
+ "canonical_resets_at_utc", "window_minutes", "limit_id", "limit_name",
4403
+ "observed_model", "account_key",
4404
+ )
4405
+
4406
+ #: The five objects whose joint presence means the summary has been maintained
4407
+ #: since it was bootstrapped. The reader uses it only when all five exist.
4408
+ _CODEX_QUOTA_LATEST_OBJECTS = (
4409
+ ("table", "codex_quota_partition_latest"),
4410
+ ("index", "idx_qws_partition_capture"),
4411
+ ("trigger", "trg_qws_latest_ins"),
4412
+ ("trigger", "trg_qws_latest_del"),
4413
+ ("trigger", "trg_qws_latest_upd"),
4414
+ )
4415
+
4416
+
4417
+ def _codex_quota_latest_raw(column, prefix: str) -> str:
4418
+ if column is None:
4419
+ return (f"COALESCE({prefix}canonical_resets_at_utc, "
4420
+ f"{prefix}resets_at_utc)")
4421
+ return f"{prefix}{column}"
4422
+
4423
+
4424
+ def _codex_quota_latest_validity_sql(prefix: str = "") -> str:
4425
+ """The loader's SQL-statable validity predicates, for one row image."""
4426
+ terms = [f"{prefix}source='codex'", f"{prefix}source_root_key IS NOT NULL"]
4427
+ terms += [f"trim(coalesce({prefix}{column}, '')) <> ''"
4428
+ for column in _CODEX_QUOTA_LATEST_REQUIRED_TEXT]
4429
+ terms += [f"unixepoch({prefix}captured_at_utc) IS NOT NULL",
4430
+ f"unixepoch({prefix}resets_at_utc) IS NOT NULL"]
4431
+ return " AND ".join(terms)
4432
+
4433
+
4434
+ def _codex_quota_latest_key_exprs(prefix: str = "") -> "tuple[str, ...]":
4435
+ return tuple(
4436
+ f"quote({_codex_quota_latest_raw(column, prefix)})"
4437
+ for _key, column in _CODEX_QUOTA_LATEST_KEYS)
4438
+
4439
+
4440
+ def _codex_quota_latest_summary_match(image: str) -> str:
4441
+ return " AND ".join(
4442
+ f"{key} = {expr}" for key, expr in zip(
4443
+ _CODEX_QUOTA_LATEST_KEY_COLUMNS,
4444
+ _codex_quota_latest_key_exprs(f"{image}.")))
4445
+
4446
+
4447
+ def _codex_quota_latest_base_match(image: str) -> str:
4448
+ return " AND ".join(
4449
+ f"{base} = {expr}" for base, expr in zip(
4450
+ _codex_quota_latest_key_exprs(""),
4451
+ _codex_quota_latest_key_exprs(f"{image}.")))
4452
+
4453
+
4454
+ def _codex_quota_latest_recompute_sql(image: str, *, only_if_absent: bool) -> str:
4455
+ """Re-derive one partition's maximum from the physical rows (one seek)."""
4456
+ absent_clause = (
4457
+ " AND NOT EXISTS (SELECT 1 FROM codex_quota_partition_latest"
4458
+ f" WHERE {_codex_quota_latest_summary_match(image)})"
4459
+ if only_if_absent else ""
4460
+ )
4461
+ return (
4462
+ "INSERT INTO codex_quota_partition_latest ("
4463
+ + ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
4464
+ + ", latest_capture_epoch)"
4465
+ f" SELECT {', '.join(_codex_quota_latest_key_exprs(image + '.'))},"
4466
+ " latest FROM (SELECT MAX(unixepoch(captured_at_utc)) AS latest"
4467
+ " FROM quota_window_snapshots"
4468
+ f" WHERE {_codex_quota_latest_validity_sql('')}"
4469
+ f" AND {_codex_quota_latest_base_match(image)})"
4470
+ f" WHERE latest IS NOT NULL{absent_clause}"
4471
+ )
4472
+
4473
+
4474
+ def _codex_quota_latest_bootstrap_select() -> str:
4475
+ """Every valid raw partition's maximum second, streamed off the seek index."""
4476
+ keys = ", ".join(_codex_quota_latest_key_exprs(""))
4477
+ return (
4478
+ f"SELECT {keys}, MAX(unixepoch(captured_at_utc))"
4479
+ " FROM quota_window_snapshots INDEXED BY idx_qws_partition_capture"
4480
+ f" WHERE {_codex_quota_latest_validity_sql('')}"
4481
+ f" GROUP BY source, {keys}"
4482
+ )
4483
+
4484
+
4485
+ def _codex_quota_latest_bootstrap_sql() -> str:
4486
+ return (
4487
+ "INSERT INTO codex_quota_partition_latest ("
4488
+ + ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
4489
+ + ", latest_capture_epoch) "
4490
+ + _codex_quota_latest_bootstrap_select()
4491
+ )
4492
+
4493
+
4494
+ def _codex_quota_latest_ddl() -> "tuple[str, ...]":
4495
+ """DDL for the summary, its seek index and its three triggers.
4496
+
4497
+ The triggers are dropped and re-created on every apply, like the change
4498
+ ledger's, so a definition change never leaves an older body behind; the
4499
+ table keeps IF NOT EXISTS and its rows survive.
4500
+ """
4501
+ columns = ",\n".join(
4502
+ f" {key} BLOB NOT NULL"
4503
+ for key in _CODEX_QUOTA_LATEST_KEY_COLUMNS)
4504
+ keys = ", ".join(_CODEX_QUOTA_LATEST_KEY_COLUMNS)
4505
+ index_columns = ", ".join((
4506
+ "source", *_codex_quota_latest_key_exprs(""),
4507
+ "unixepoch(captured_at_utc)"))
4508
+ insert_new = (
4509
+ f"INSERT INTO codex_quota_partition_latest ({keys}, latest_capture_epoch)"
4510
+ f" SELECT {', '.join(_codex_quota_latest_key_exprs('NEW.'))},"
4511
+ " unixepoch(NEW.captured_at_utc)"
4512
+ " WHERE NOT EXISTS (SELECT 1 FROM codex_quota_partition_latest"
4513
+ f" WHERE {_codex_quota_latest_summary_match('NEW')})"
4514
+ )
4515
+ return (
4516
+ "CREATE TABLE IF NOT EXISTS codex_quota_partition_latest (\n"
4517
+ f"{columns},\n"
4518
+ " latest_capture_epoch BLOB NOT NULL,\n"
4519
+ f" PRIMARY KEY ({keys})\n"
4520
+ " ) WITHOUT ROWID",
4521
+ "CREATE INDEX IF NOT EXISTS idx_qws_partition_capture "
4522
+ f"ON quota_window_snapshots({index_columns}) "
4523
+ f"WHERE {_codex_quota_latest_validity_sql('')}",
4524
+ "DROP TRIGGER IF EXISTS trg_qws_latest_ins",
4525
+ "CREATE TRIGGER trg_qws_latest_ins\n"
4526
+ " AFTER INSERT ON quota_window_snapshots\n"
4527
+ f" WHEN {_codex_quota_latest_validity_sql('NEW.')}\n"
4528
+ " BEGIN\n"
4529
+ " DELETE FROM codex_quota_partition_latest"
4530
+ f" WHERE {_codex_quota_latest_summary_match('NEW')}"
4531
+ " AND latest_capture_epoch < unixepoch(NEW.captured_at_utc);\n"
4532
+ f" {insert_new};\n"
4533
+ " END",
4534
+ "DROP TRIGGER IF EXISTS trg_qws_latest_del",
4535
+ "CREATE TRIGGER trg_qws_latest_del\n"
4536
+ " AFTER DELETE ON quota_window_snapshots\n"
4537
+ f" WHEN {_codex_quota_latest_validity_sql('OLD.')}\n"
4538
+ " BEGIN\n"
4539
+ " DELETE FROM codex_quota_partition_latest"
4540
+ f" WHERE {_codex_quota_latest_summary_match('OLD')}"
4541
+ " AND latest_capture_epoch = unixepoch(OLD.captured_at_utc);\n"
4542
+ " "
4543
+ f"{_codex_quota_latest_recompute_sql('OLD', only_if_absent=True)};\n"
4544
+ " END",
4545
+ "DROP TRIGGER IF EXISTS trg_qws_latest_upd",
4546
+ "CREATE TRIGGER trg_qws_latest_upd\n"
4547
+ f" AFTER UPDATE OF {', '.join(_CODEX_QUOTA_LATEST_UPDATE_OF)}"
4548
+ " ON quota_window_snapshots\n"
4549
+ " WHEN OLD.source = 'codex' OR NEW.source = 'codex'\n"
4550
+ " BEGIN\n"
4551
+ " DELETE FROM codex_quota_partition_latest"
4552
+ f" WHERE {_codex_quota_latest_summary_match('OLD')};\n"
4553
+ " "
4554
+ f"{_codex_quota_latest_recompute_sql('OLD', only_if_absent=False)};\n"
4555
+ " DELETE FROM codex_quota_partition_latest"
4556
+ f" WHERE {_codex_quota_latest_summary_match('NEW')};\n"
4557
+ " "
4558
+ f"{_codex_quota_latest_recompute_sql('NEW', only_if_absent=False)};\n"
4559
+ " END",
4560
+ )
4561
+
4562
+
4563
+ def _apply_codex_quota_latest_summary(
4564
+ conn: sqlite3.Connection, *, rebootstrap: bool = False,
4565
+ ) -> None:
4566
+ """Install the latest-capture summary and keep it equal to the physical rows.
4567
+
4568
+ Atomic: the table, its seek index, the three triggers and, when needed, the
4569
+ bootstrap commit together — as a savepoint inside the caller's open
4570
+ transaction, or as an IMMEDIATE transaction of their own. The bootstrap runs
4571
+ when any of the five objects was missing before this call (so a store that
4572
+ carries all five has been maintained by the triggers since they were
4573
+ created) or when ``rebootstrap`` asks for it, which migration 047 always
4574
+ does, so a re-upgrade after an older binary trimmed its marker re-derives
4575
+ the summary from the physical rows.
4576
+
4577
+ Guarded on the three columns the partition reads: a legacy-shape cache whose
4578
+ schema apply took the FTS early-return before they existed keeps the
4579
+ window-query fallback, which is correct, just not spill-free.
4580
+ """
4581
+ columns = {
4582
+ str(row[1]) for row in conn.execute(
4583
+ "PRAGMA table_info(quota_window_snapshots)")
4584
+ }
4585
+ if not {"canonical_resets_at_utc", "observed_model", "account_key"} <= columns:
4586
+ return
4587
+ own_transaction = not conn.in_transaction
4588
+ conn.execute(
4589
+ "BEGIN IMMEDIATE" if own_transaction
4590
+ else "SAVEPOINT codex_quota_latest_apply")
4591
+ try:
4592
+ present = {
4593
+ (str(row[0]), str(row[1])) for row in conn.execute(
4594
+ "SELECT type, name FROM sqlite_master "
4595
+ "WHERE type IN ('table', 'index', 'trigger')")
4596
+ }
4597
+ complete = all(item in present for item in _CODEX_QUOTA_LATEST_OBJECTS)
4598
+ for statement in _codex_quota_latest_ddl():
4599
+ conn.execute(statement)
4600
+ if rebootstrap or not complete:
4601
+ conn.execute("DELETE FROM codex_quota_partition_latest")
4602
+ conn.execute(_codex_quota_latest_bootstrap_sql())
4603
+ except BaseException:
4604
+ if own_transaction:
4605
+ conn.rollback()
4606
+ else:
4607
+ conn.execute("ROLLBACK TO codex_quota_latest_apply")
4608
+ conn.execute("RELEASE codex_quota_latest_apply")
4609
+ raise
4610
+ if own_transaction:
4611
+ conn.commit()
4612
+ else:
4613
+ conn.execute("RELEASE codex_quota_latest_apply")
4614
+
4615
+
4616
+ def _apply_codex_quota_load_order_index(conn: sqlite3.Connection) -> None:
4617
+ """The general loader's ORDER BY, served by an index (#901 §5.1, T1/T3).
4618
+
4619
+ Adoption (W3a, every Codex ingest including the hook), breakdown evidence,
4620
+ the projection's root loads and the CLI commands all read complete
4621
+ populations in ``source_root_key, captured_at_utc, resets_at_utc,
4622
+ source_path, line_offset`` order; without this index each one sorted its
4623
+ whole population through a temp b-tree. ``canonical_resets_at_utc`` rides
4624
+ at the end so the adoption pass's canonical-reset bound is evaluated on the
4625
+ index entry before any table row is fetched. Guarded on that column like
4626
+ idx_qws_physical_group.
4627
+ """
4628
+ columns = {
4629
+ str(row[1]) for row in conn.execute(
4630
+ "PRAGMA table_info(quota_window_snapshots)")
4631
+ }
4632
+ if "canonical_resets_at_utc" not in columns:
4633
+ return
4634
+ conn.execute(
4635
+ "CREATE INDEX IF NOT EXISTS idx_qws_codex_load_order "
4636
+ "ON quota_window_snapshots(source_root_key, captured_at_utc, "
4637
+ "resets_at_utc, source_path, line_offset, canonical_resets_at_utc) "
4638
+ "WHERE source='codex'"
4639
+ )
4640
+
4641
+
4642
+ def _apply_codex_metadata_lookup_indexes(conn: sqlite3.Connection) -> None:
4643
+ """The W2 correlated lookups' indexes (#901 §5.2, T2).
4644
+
4645
+ ``idx_codex_entries_root_path_time`` answers each file's and each thread's
4646
+ first accounting instant with one covering seek (the existing
4647
+ ``idx_codex_entries_root_path`` keeps serving the minimum ``id``, because
4648
+ the rowid trails its two columns). ``idx_codex_files_alias_recent`` and
4649
+ ``idx_codex_threads_recent`` hold the two queries' final orders so neither
4650
+ sorts. Each is guarded on its columns, because a legacy-shape cache can
4651
+ lack the ALTER-added ones.
4652
+ """
4653
+ def _columns(table: str) -> "set[str]":
4654
+ return {
4655
+ str(row[0]) for row in conn.execute(
4656
+ "SELECT name FROM pragma_table_info(?)", (table,))
4657
+ }
4658
+
4659
+ if {"source_root_key", "source_path", "timestamp_utc"} <= _columns(
4660
+ "codex_session_entries"):
4661
+ conn.execute(
4662
+ "CREATE INDEX IF NOT EXISTS idx_codex_entries_root_path_time "
4663
+ "ON codex_session_entries(source_root_key, source_path, "
4664
+ "timestamp_utc)"
4665
+ )
4666
+ if {"last_ingested_at", "path", "last_native_thread_id"} <= _columns(
4667
+ "codex_session_files"):
4668
+ conn.execute(
4669
+ "CREATE INDEX IF NOT EXISTS idx_codex_files_alias_recent "
4670
+ "ON codex_session_files(last_ingested_at, path) "
4671
+ "WHERE last_native_thread_id IS NOT NULL "
4672
+ "AND last_native_thread_id != ''"
4673
+ )
4674
+ if {"last_seen_utc", "conversation_key"} <= _columns(
4675
+ "codex_conversation_threads"):
4676
+ conn.execute(
4677
+ "CREATE INDEX IF NOT EXISTS idx_codex_threads_recent "
4678
+ "ON codex_conversation_threads(last_seen_utc, conversation_key)"
4679
+ )
4680
+
4681
+
4682
+ def _apply_pricing_model_indexes(conn: sqlite3.Connection) -> None:
4683
+ """Model-first covering indexes for ``_pricing_observed_models`` (#901 §5.3).
4684
+
4685
+ Both of its modes — doctor's trailing 30 days and pricing-check's whole
4686
+ history — group by model over one entries table. Covering the token columns
4687
+ lets SQLite stream the GROUP BY off the index with the timestamp filter
4688
+ evaluated on the entry; a narrower ``(model, timestamp_utc)`` index streams
4689
+ too but pays one table lookup per row, a latency regression the §6.3
4690
+ per-call-site gate would catch.
4691
+ """
4692
+ def _columns(table: str) -> "set[str]":
4693
+ return {
4694
+ str(row[0]) for row in conn.execute(
4695
+ "SELECT name FROM pragma_table_info(?)", (table,))
4696
+ }
4697
+
4698
+ if {"model", "timestamp_utc", "input_tokens", "output_tokens",
4699
+ "cache_create_tokens", "cache_read_tokens"} <= _columns(
4700
+ "session_entries"):
4701
+ conn.execute(
4702
+ "CREATE INDEX IF NOT EXISTS idx_entries_model_time "
4703
+ "ON session_entries(model, timestamp_utc, input_tokens, "
4704
+ "output_tokens, cache_create_tokens, cache_read_tokens)"
4705
+ )
4706
+ if {"model", "timestamp_utc", "total_tokens"} <= _columns(
4707
+ "codex_session_entries"):
4708
+ conn.execute(
4709
+ "CREATE INDEX IF NOT EXISTS idx_codex_entries_model_time "
4710
+ "ON codex_session_entries(model, timestamp_utc, total_tokens)"
4711
+ )
4712
+
4713
+
4714
+ def _apply_codex_window_attribution_read_index(conn: sqlite3.Connection) -> None:
4715
+ """The window-attribution read's order, served by an index (#901 I1/G1).
4716
+
4717
+ ``_cctally_cache.load_active_window_attributions`` (every Codex ingest's
4718
+ spend reconciliation, so the dashboard and the Codex hook, and the
4719
+ attribution overlay of every quota load) returns every asserted or
4720
+ retracted weekly attribution in ``asserted_at_utc, op_id`` order; without
4721
+ this index it sorted all of them through a temp b-tree, a population that
4722
+ grows by one assertion per attributed window. ``window_minutes`` leads
4723
+ because the read's one equality filter is on it; the root, retraction and
4724
+ account predicates are evaluated along the ordered scan. ``source_root_key``
4725
+ is deliberately NOT ahead of the ordering columns: several roots share one
4726
+ global assertion order (Amendment 1 item 1, ``c901-design-2``). Guarded on
4727
+ the table's columns, because the conversations store drops the table and a
4728
+ legacy-shape cache can predate it.
4729
+ """
4730
+ columns = {
4731
+ str(row[0]) for row in conn.execute(
4732
+ "SELECT name FROM pragma_table_info(?)",
4733
+ ("codex_window_attributions",))
4734
+ }
4735
+ if {"window_minutes", "asserted_at_utc", "op_id"} <= columns:
4736
+ conn.execute(
4737
+ "CREATE INDEX IF NOT EXISTS idx_codex_window_attributions_read_order "
4738
+ "ON codex_window_attributions(window_minutes ASC, asserted_at_utc ASC, "
4739
+ "op_id ASC)"
4740
+ )
4741
+
4742
+
4286
4743
  # === Region 7b2: Eager cache-migration trigger (V4 — same-invocation 008 apply) ===
4287
4744
 
4288
4745
 
@@ -4451,7 +4908,7 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
4451
4908
  "031_codex_file_account_map"),
4452
4909
  SchemaDeliveryObject(
4453
4910
  "index", "idx_qws_physical_group", "_apply_codex_quota_group_index",
4454
- "040_codex_quota_physical_group_index"),
4911
+ "048_codex_quota_physical_group_order"),
4455
4912
  SchemaDeliveryObject(
4456
4913
  "index", "idx_qws_unresolved_model",
4457
4914
  "_apply_codex_quota_unresolved_model_index",
@@ -4537,6 +4994,43 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
4537
4994
  "trigger", "trg_codex_accounting_upd",
4538
4995
  "_apply_codex_accounting_change_ledger",
4539
4996
  "044_codex_accounting_change_ledger"),
4997
+ SchemaDeliveryObject(
4998
+ "table", "codex_quota_partition_latest",
4999
+ "_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
5000
+ SchemaDeliveryObject(
5001
+ "index", "idx_qws_partition_capture",
5002
+ "_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
5003
+ SchemaDeliveryObject(
5004
+ "trigger", "trg_qws_latest_del",
5005
+ "_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
5006
+ SchemaDeliveryObject(
5007
+ "trigger", "trg_qws_latest_ins",
5008
+ "_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
5009
+ SchemaDeliveryObject(
5010
+ "trigger", "trg_qws_latest_upd",
5011
+ "_apply_codex_quota_latest_summary", "047_spill_free_read_paths"),
5012
+ SchemaDeliveryObject(
5013
+ "index", "idx_qws_codex_load_order",
5014
+ "_apply_codex_quota_load_order_index", "047_spill_free_read_paths"),
5015
+ SchemaDeliveryObject(
5016
+ "index", "idx_codex_entries_root_path_time",
5017
+ "_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
5018
+ SchemaDeliveryObject(
5019
+ "index", "idx_codex_files_alias_recent",
5020
+ "_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
5021
+ SchemaDeliveryObject(
5022
+ "index", "idx_codex_threads_recent",
5023
+ "_apply_codex_metadata_lookup_indexes", "047_spill_free_read_paths"),
5024
+ SchemaDeliveryObject(
5025
+ "index", "idx_entries_model_time",
5026
+ "_apply_pricing_model_indexes", "047_spill_free_read_paths"),
5027
+ SchemaDeliveryObject(
5028
+ "index", "idx_codex_entries_model_time",
5029
+ "_apply_pricing_model_indexes", "047_spill_free_read_paths"),
5030
+ SchemaDeliveryObject(
5031
+ "index", "idx_codex_window_attributions_read_order",
5032
+ "_apply_codex_window_attribution_read_index",
5033
+ "047_spill_free_read_paths"),
4540
5034
  )
4541
5035
 
4542
5036
 
@@ -5249,6 +5743,15 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
5249
5743
  _apply_codex_quota_unresolved_model_index(conn)
5250
5744
  _apply_codex_quota_change_ledger(conn)
5251
5745
  _apply_codex_accounting_change_ledger(conn)
5746
+ # #901 (spec §5.1-§5.3): the latest-capture summary, the general loader's
5747
+ # ordering index, the W2 lookup indexes, the pricing model indexes and the
5748
+ # window-attribution read order (Amendment 1 item 1). BEFORE the
5749
+ # legacy-FTS early-return, so an old-shape cache.db gains them.
5750
+ _apply_codex_quota_latest_summary(conn)
5751
+ _apply_codex_quota_load_order_index(conn)
5752
+ _apply_codex_metadata_lookup_indexes(conn)
5753
+ _apply_pricing_model_indexes(conn)
5754
+ _apply_codex_window_attribution_read_index(conn)
5252
5755
  conn.execute(
5253
5756
  "CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
5254
5757
  "ON session_files(session_id)"
@@ -5394,6 +5897,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
5394
5897
  _apply_codex_find_projection_schema(conn)
5395
5898
  _apply_conversation_generation_schema(conn)
5396
5899
  _apply_codex_conversation_source_identity(conn)
5900
+ _apply_conversation_latest_meta_indexes(conn)
5397
5901
  return
5398
5902
 
5399
5903
  _apply_cache_schema(conn)
@@ -5425,6 +5929,10 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
5425
5929
  -- tests/test_conversations_transcripts_only.py, which derives its
5426
5930
  -- expectation from COVERAGE_CACHE_FAMILIES rather than restating a list.
5427
5931
  DROP TABLE IF EXISTS codex_window_attributions;
5932
+ -- #901: the latest-capture summary is a cache.db read model over
5933
+ -- quota_window_snapshots, which was dropped above together with its
5934
+ -- triggers and seek index. Drop the table as well.
5935
+ DROP TABLE IF EXISTS codex_quota_partition_latest;
5428
5936
 
5429
5937
  CREATE TABLE IF NOT EXISTS conversation_source_files (
5430
5938
  path TEXT PRIMARY KEY,
@@ -5478,6 +5986,7 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
5478
5986
  _apply_codex_find_projection_schema(conn)
5479
5987
  _apply_conversation_generation_schema(conn)
5480
5988
  _apply_codex_conversation_source_identity(conn)
5989
+ _apply_conversation_latest_meta_indexes(conn)
5481
5990
 
5482
5991
 
5483
5992
  def _ensure_codex_session_meta_provenance_index(
@@ -5654,6 +6163,36 @@ def _apply_codex_conversation_source_identity(conn: sqlite3.Connection) -> None:
5654
6163
  conn, "codex_conversation_source_files", "inode", "INTEGER")
5655
6164
 
5656
6165
 
6166
+ def _apply_conversation_latest_meta_indexes(conn: sqlite3.Connection) -> None:
6167
+ """The latest-metadata lookups' partial indexes (#901 Q11, 901-PA-001 b).
6168
+
6169
+ ``_lib_conversation_query._session_latest_meta_map`` resolves a session's
6170
+ latest non-NULL ``cwd`` and latest non-NULL ``git_branch`` with two
6171
+ ``LIMIT 1`` lookups ordered ``timestamp_utc DESC, id DESC``. Each partial
6172
+ index holds exactly the rows its lookup may return, in that order, so the
6173
+ lookup is one seek however long the session is and however sparse its
6174
+ metadata. The window query they replace sorted every touched session's
6175
+ whole history through temp b-trees on each conversation-sync pass (9.56 MB
6176
+ of temp in one measured call over a long session).
6177
+
6178
+ Called by the fresh-install schema apply and by conversations migration
6179
+ 011, so both shapes converge. Skipped when ``conversation_messages`` is
6180
+ absent (a reachable state, not a corrupt one).
6181
+ """
6182
+ if not _table_exists(conn, "conversation_messages"):
6183
+ return
6184
+ conn.execute(
6185
+ "CREATE INDEX IF NOT EXISTS idx_conv_session_latest_cwd "
6186
+ "ON conversation_messages(session_id, timestamp_utc, id) "
6187
+ "WHERE cwd IS NOT NULL"
6188
+ )
6189
+ conn.execute(
6190
+ "CREATE INDEX IF NOT EXISTS idx_conv_session_latest_git_branch "
6191
+ "ON conversation_messages(session_id, timestamp_utc, id) "
6192
+ "WHERE git_branch IS NOT NULL"
6193
+ )
6194
+
6195
+
5657
6196
  def _apply_codex_find_projection_schema(conn: sqlite3.Connection) -> None:
5658
6197
  """Create #482's disposable visible-text projection tables.
5659
6198
 
@@ -5817,6 +6356,14 @@ CONVERSATIONS_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
5817
6356
  "column", "codex_conversation_source_files.inode",
5818
6357
  "_apply_codex_conversation_source_identity",
5819
6358
  "010_codex_conversation_source_file_identity"),
6359
+ SchemaDeliveryObject(
6360
+ "index", "idx_conv_session_latest_cwd",
6361
+ "_apply_conversation_latest_meta_indexes",
6362
+ "011_conversation_latest_meta_indexes"),
6363
+ SchemaDeliveryObject(
6364
+ "index", "idx_conv_session_latest_git_branch",
6365
+ "_apply_conversation_latest_meta_indexes",
6366
+ "011_conversation_latest_meta_indexes"),
5820
6367
  SchemaDeliveryObject(
5821
6368
  "table", "claude_conversation_account_stamp_gaps",
5822
6369
  "_apply_conversation_generation_schema",
@@ -6453,6 +7000,36 @@ def _conv_010_codex_conversation_source_file_identity(
6453
7000
  conn.commit()
6454
7001
 
6455
7002
 
7003
+ @conversations_migration("011_conversation_latest_meta_indexes")
7004
+ def _conv_011_conversation_latest_meta_indexes(
7005
+ conn: sqlite3.Connection,
7006
+ ) -> None:
7007
+ """Deliver the latest-metadata partial indexes (#901 Q11, 901-PA-001 b).
7008
+
7009
+ ``_session_latest_meta_map`` now resolves each touched session's latest
7010
+ ``cwd`` and ``git_branch`` with two index-served ``LIMIT 1`` lookups; the
7011
+ indexes and the reasoning live on ``_apply_conversation_latest_meta_indexes``,
7012
+ the helper the fresh-install apply also calls. The schema apply that would
7013
+ create them is version-gated, so this registration is what reaches an
7014
+ already-current store.
7015
+
7016
+ Takes the Claude provider flock and defers cleanly on contention, like 009:
7017
+ building the indexes holds SQLite's write lock for a sort of the Claude
7018
+ transcript rows (one-time setup work), and a concurrent Claude transcript
7019
+ writer then reports contention instead of exhausting its busy timeout.
7020
+
7021
+ Re-running is a no-op (``IF NOT EXISTS``). NO self-stamp — the dispatcher
7022
+ central-stamps on a clean return (#140).
7023
+ """
7024
+ held = _acquire_conversations_db_claude_provider_flock(
7025
+ conn, migration="conversations 011 latest metadata indexes")
7026
+ try:
7027
+ _apply_conversation_latest_meta_indexes(conn)
7028
+ conn.commit()
7029
+ finally:
7030
+ _release_cache_db_writer_flocks(held)
7031
+
7032
+
6456
7033
  # #177 S6: the consolidated multi-column external-content FTS5 table that
6457
7034
  # replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
6458
7035
  # pair. The three column names MUST match the conversation_messages columns BY
@@ -9277,6 +9854,70 @@ def _046_codex_source_file_identity(conn: sqlite3.Connection) -> None:
9277
9854
  conn.commit()
9278
9855
 
9279
9856
 
9857
+ @cache_migration("047_spill_free_read_paths")
9858
+ def _047_spill_free_read_paths(conn: sqlite3.Connection) -> None:
9859
+ """#901 (spec §5.1-§5.3): deliver the spill-free read paths to existing stores.
9860
+
9861
+ ``_apply_cache_schema`` creates every object below for a new store, but that
9862
+ pass is VERSION-GATED: ``open_cache_db`` runs it only when ``user_version``
9863
+ differs from ``len(_CACHE_MIGRATIONS)``. Registering here bumps the head,
9864
+ the mechanism migrations 029, 031, 044, 045 and 046 rely on.
9865
+
9866
+ The handler ALWAYS re-derives the latest-capture summary from the physical
9867
+ rows (``rebootstrap=True``), so a re-upgrade after an older binary trimmed
9868
+ this marker converges on the rows actually stored whatever happened to the
9869
+ summary meanwhile. On a first upgrade the schema apply has already
9870
+ bootstrapped it once; the second derivation is one ordered scan of the new
9871
+ seek index, paid once.
9872
+
9873
+ It also delivers ``idx_codex_window_attributions_read_order`` (Amendment 1
9874
+ item 1, ``c901-design-2``), the window-attribution read's assertion order.
9875
+
9876
+ Takes the Codex provider flock like handlers 024-027, 034 and 043, because
9877
+ the summary is Codex-derived, and DEFERS (``MigrationGateNotMet``) before
9878
+ any DDL when a Codex sync holds it. Idempotent: every object uses IF NOT
9879
+ EXISTS or drop-then-create, and a re-derivation over its own output writes
9880
+ the same rows. NO self-stamp — the dispatcher central-stamps on a clean
9881
+ return (#140).
9882
+ """
9883
+ held = _acquire_cache_db_codex_provider_flock(
9884
+ conn, migration="047 spill-free read paths")
9885
+ try:
9886
+ _apply_codex_quota_latest_summary(conn, rebootstrap=True)
9887
+ _apply_codex_quota_load_order_index(conn)
9888
+ _apply_codex_metadata_lookup_indexes(conn)
9889
+ _apply_pricing_model_indexes(conn)
9890
+ _apply_codex_window_attribution_read_index(conn)
9891
+ conn.commit()
9892
+ finally:
9893
+ _release_cache_db_writer_flocks(held)
9894
+
9895
+
9896
+ @cache_migration("048_codex_quota_physical_group_order")
9897
+ def _048_codex_quota_physical_group_order(conn: sqlite3.Connection) -> None:
9898
+ """#901 Q11 (spec §5.3a, 901-PA-001 a): stream every physical-group shard.
9899
+
9900
+ Replaces ``idx_qws_physical_group``'s five-member definition with the same
9901
+ name, partial predicate and five equality members followed by the loader's
9902
+ order columns ``captured_at_utc, resets_at_utc, source_path,
9903
+ line_offset``. The shard SQL, its population and its order are unchanged;
9904
+ only the sorter goes. The reasoning lives on
9905
+ ``_apply_codex_quota_group_index``, which this delegates to so the
9906
+ migration and the schema apply cannot drift, and which replaces the index
9907
+ atomically.
9908
+
9909
+ Same version-gate reason as 040: the schema apply that converges the index
9910
+ is skipped by a steady-state open, so registering here is what makes an
9911
+ already-current install pick it up. The rebuild is a one-time sort of the
9912
+ Codex quota rows (setup work, not a recurring path).
9913
+
9914
+ Re-running is a no-op: a current index is left alone. NO self-stamp — the
9915
+ dispatcher central-stamps on a clean return (#140).
9916
+ """
9917
+ _apply_codex_quota_group_index(conn)
9918
+ conn.commit()
9919
+
9920
+
9280
9921
  # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
9281
9922
 
9282
9923
  @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
@@ -10867,6 +11508,13 @@ def _repair_marker_path(path: pathlib.Path) -> pathlib.Path:
10867
11508
  return path.with_name(f"{path.name}.repairing")
10868
11509
 
10869
11510
 
11511
+ def _keeper_yield_request_path(path: pathlib.Path) -> pathlib.Path:
11512
+ """#901 W9 (Q15): the store-scoped request that makes every process's
11513
+ idle checkpoint keeper (`_lib_wal_checkpoint`) close. Its owner record is
11514
+ the repair marker's, written, read and checked for liveness the same way."""
11515
+ return path.with_name(f"{path.name}.keeper-yield")
11516
+
11517
+
10870
11518
  _REPAIR_OWNER_VERSION = 1
10871
11519
  _REPAIR_OWNER_UNKNOWN_IDENTITY_LEASE_SECONDS = 30 * 60
10872
11520
 
@@ -11019,10 +11667,30 @@ def _claim_repair_marker(
11019
11667
  path: pathlib.Path,
11020
11668
  ) -> "tuple[RepairMarkerClaim | None, str]":
11021
11669
  """Atomically publish a complete owner record; reclaim stale ownership."""
11022
- marker = _repair_marker_path(path)
11670
+ return _claim_owner_record(
11671
+ _repair_marker_path(path),
11672
+ lambda: _remove_stale_repair_marker(path),
11673
+ owner="repair-owner",
11674
+ record="repair marker",
11675
+ )
11676
+
11677
+
11678
+ def _claim_owner_record(
11679
+ marker: pathlib.Path,
11680
+ remove_stale: "Callable[[], tuple[bool, str]]",
11681
+ *,
11682
+ owner: str,
11683
+ record: str,
11684
+ ) -> "tuple[RepairMarkerClaim | None, str]":
11685
+ """Publish ``marker`` with a complete owner record by exclusive create.
11686
+
11687
+ The repair marker's discipline, shared with #901's keeper-yield request:
11688
+ the record is written and fsynced under a private name and linked into
11689
+ place, so it is complete or absent; an existing record is replaced only
11690
+ when ``remove_stale`` proves its owner dead or its pid reused."""
11023
11691
  process_start = _process_start_identity(os.getpid())
11024
11692
  if process_start is None:
11025
- raise OSError("could not establish repair-owner process identity")
11693
+ raise OSError(f"could not establish {owner} process identity")
11026
11694
  for _attempt in range(3):
11027
11695
  claim_id = f"{os.getpid()}-{time.time_ns()}-{os.urandom(8).hex()}"
11028
11696
  temp = marker.with_name(f".{marker.name}.{claim_id}.tmp")
@@ -11041,7 +11709,7 @@ def _claim_repair_marker(
11041
11709
  try:
11042
11710
  os.link(temp, marker)
11043
11711
  except FileExistsError:
11044
- removed, reason = _remove_stale_repair_marker(path)
11712
+ removed, reason = remove_stale()
11045
11713
  if not removed:
11046
11714
  return None, reason
11047
11715
  continue
@@ -11052,7 +11720,7 @@ def _claim_repair_marker(
11052
11720
  temp.unlink()
11053
11721
  except FileNotFoundError:
11054
11722
  pass
11055
- return None, f"could not claim repair marker {marker}"
11723
+ return None, f"could not claim {record} {marker}"
11056
11724
 
11057
11725
 
11058
11726
  def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None:
@@ -11067,8 +11735,135 @@ def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None
11067
11735
  _fsync_directory(marker.parent)
11068
11736
 
11069
11737
 
11738
+ #: #901 W9 (Q15): the one monotonic deadline a process that needs a
11739
+ #: cache.db or conversations.db family exclusive or drained waits for the
11740
+ #: other processes' idle checkpoint keepers to close, per store: twice the
11741
+ #: keepers' 10 s timer interval. Never renewed per keeper, scan or retry.
11742
+ _KEEPER_YIELD_DEADLINE_SECONDS = 20.0
11743
+ #: The pause between two handle checks, or two exclusive acquisitions, while
11744
+ #: a requester waits under that deadline.
11745
+ _KEEPER_YIELD_RETRY_SECONDS = 0.25
11746
+
11747
+
11748
+ def _remove_stale_keeper_yield_request(
11749
+ path: pathlib.Path,
11750
+ ) -> "tuple[bool, str]":
11751
+ """Remove a provably stale keeper-yield request (dead or reused-pid
11752
+ owner), as `_remove_stale_repair_marker` does for the repair marker."""
11753
+ request = _keeper_yield_request_path(path)
11754
+ if not request.exists():
11755
+ return True, ""
11756
+ live, reason = _repair_marker_is_live(request)
11757
+ if live:
11758
+ return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
11759
+ try:
11760
+ request.unlink()
11761
+ except FileNotFoundError:
11762
+ return True, ""
11763
+ except OSError as exc:
11764
+ return False, f"could not remove stale keeper-yield request {request}: {exc}"
11765
+ _fsync_directory(request.parent)
11766
+ return True, reason
11767
+
11768
+
11769
+ def _claim_keeper_yield_request(
11770
+ path: pathlib.Path,
11771
+ ) -> "tuple[RepairMarkerClaim | None, str]":
11772
+ """Publish ``<store>.keeper-yield`` with this process as its owner."""
11773
+ return _claim_owner_record(
11774
+ _keeper_yield_request_path(path),
11775
+ lambda: _remove_stale_keeper_yield_request(path),
11776
+ owner="keeper-yield-owner",
11777
+ record="keeper-yield request",
11778
+ )
11779
+
11780
+
11781
+ def _release_keeper_yield_request(
11782
+ path: pathlib.Path, claim: RepairMarkerClaim,
11783
+ ) -> None:
11784
+ """Remove the request; only its owner does, and a record someone else now
11785
+ owns is left alone. Best effort: a request left behind fences keepers
11786
+ only while this process lives, and it is stale once the process exits."""
11787
+ request = _keeper_yield_request_path(path)
11788
+ owner = _read_repair_owner(request)
11789
+ if owner is not None and owner.get("claim_id") != claim.claim_id:
11790
+ return
11791
+ try:
11792
+ request.unlink()
11793
+ except OSError:
11794
+ return
11795
+ try:
11796
+ _fsync_directory(request.parent)
11797
+ except OSError:
11798
+ pass
11799
+
11800
+
11801
+ @contextlib.contextmanager
11802
+ def _keeper_yield_request(path) -> "Iterator[float]":
11803
+ """Ask every process's idle checkpoint keeper on the store to close (#901
11804
+ W9, Q15) and yield the one monotonic deadline to wait for them.
11805
+
11806
+ The caller already holds its maintenance and provider locks, taken as
11807
+ before and failing as promptly as before; the request grants no authority
11808
+ of its own. It is published with the repair marker's owner record, this
11809
+ process's own keeper is closed at once, and the request is removed when
11810
+ the caller finishes or fails. The other processes observe it at their next
11811
+ `arm`, timer round or exit and close their keepers without a checkpoint.
11812
+ When it cannot be published the caller still waits, because a sync or a
11813
+ reader may close in time; it then refuses at the deadline as before."""
11814
+ path = pathlib.Path(path)
11815
+ try:
11816
+ claim, _reason = _claim_keeper_yield_request(path)
11817
+ except OSError:
11818
+ claim = None
11819
+ try:
11820
+ try:
11821
+ import _lib_wal_checkpoint
11822
+
11823
+ _lib_wal_checkpoint.release(path)
11824
+ except Exception: # noqa: BLE001 — the drain check below decides
11825
+ pass
11826
+ yield time.monotonic() + _KEEPER_YIELD_DEADLINE_SECONDS
11827
+ finally:
11828
+ if claim is not None:
11829
+ _release_keeper_yield_request(path, claim)
11830
+
11831
+
11832
+ def _await_family_drained(
11833
+ path: pathlib.Path, deadline: float,
11834
+ ) -> "set[int] | None":
11835
+ """Repeat the family's handle check until it finds no open handle or the
11836
+ deadline passes (#901 W9, Q15); return the last answer.
11837
+
11838
+ ``None`` (unknown) returns at once, as a refusal. Nothing is killed,
11839
+ ignored or exempted: at the deadline the caller refuses with today's
11840
+ message for whatever handle remains — a keeper that did not yield, a sync
11841
+ in progress, a reader."""
11842
+ while True:
11843
+ open_pids = _db_family_open_pids(path)
11844
+ if not open_pids:
11845
+ return open_pids
11846
+ remaining = deadline - time.monotonic()
11847
+ if remaining <= 0:
11848
+ return open_pids
11849
+ time.sleep(min(_KEEPER_YIELD_RETRY_SECONDS, remaining))
11850
+
11851
+
11070
11852
  def _db_family_open_pids(path: pathlib.Path) -> "set[int] | None":
11071
- """Return processes with any SQLite family member open; None if unknown."""
11853
+ """Return processes with any SQLite family member open; None if unknown.
11854
+
11855
+ #901 W9 (Q14): this process's idle keeper for the store is released first,
11856
+ without a checkpoint. It exists only to defer checkpoints, and every
11857
+ caller asks this question before replacing or quarantining the family, so
11858
+ the keeper must never be the handle that refuses this process's own store
11859
+ recovery; the next sync's arm reopens it on whatever file is then there.
11860
+ """
11861
+ try:
11862
+ import _lib_wal_checkpoint
11863
+
11864
+ _lib_wal_checkpoint.release(path)
11865
+ except Exception: # noqa: BLE001 — the drain check below still decides
11866
+ pass
11072
11867
  family = [
11073
11868
  pathlib.Path(str(path) + suffix)
11074
11869
  for suffix in ("", "-journal", "-wal", "-shm")
@@ -12065,13 +12860,51 @@ def _vacuum_required_free_bytes(path) -> int:
12065
12860
  return 2 * db_bytes + wal_bytes
12066
12861
 
12067
12862
 
12068
- def _run_vacuum_exclusive(path, label: str) -> int:
12863
+ def _acquire_vacuum_exclusion(
12864
+ conn: sqlite3.Connection, deadline: "float | None",
12865
+ ) -> None:
12866
+ """Take a WAL store's EXCLUSIVE lock and keep it (#901 W9, Q15).
12867
+
12868
+ Under ``locking_mode=EXCLUSIVE`` a write transaction takes the database
12869
+ file's EXCLUSIVE lock and the connection keeps it after the transaction
12870
+ ends, so ``BEGIN IMMEDIATE`` then ``ROLLBACK`` acquires the exclusion
12871
+ without writing anything, and the checkpoint and VACUUM that follow run
12872
+ under one continuous hold (#386). Only this acquisition is retried — each
12873
+ attempt under the existing busy timeout, a short pause between attempts —
12874
+ until ``deadline``, the keepers' drain grace; the VACUUM is never retried.
12875
+ ``deadline=None`` makes one attempt. The last busy error is raised at the
12876
+ deadline, so the caller's refusal is today's."""
12877
+ while True:
12878
+ try:
12879
+ conn.execute("BEGIN IMMEDIATE")
12880
+ except sqlite3.OperationalError as exc:
12881
+ text = str(exc).lower()
12882
+ if (
12883
+ deadline is None
12884
+ or ("lock" not in text and "busy" not in text)
12885
+ or time.monotonic() >= deadline
12886
+ ):
12887
+ raise
12888
+ time.sleep(max(0.0, min(_KEEPER_YIELD_RETRY_SECONDS,
12889
+ deadline - time.monotonic())))
12890
+ continue
12891
+ conn.execute("ROLLBACK")
12892
+ return
12893
+
12894
+
12895
+ def _run_vacuum_exclusive(
12896
+ path, label: str, *, deadline: "float | None" = None,
12897
+ ) -> int:
12069
12898
  """VACUUM ``path`` under EXCLUSIVE; checkpoint WAL-backed stores first.
12070
12899
 
12071
12900
  ``locking_mode=EXCLUSIVE`` + a short ``busy_timeout`` make a concurrent
12072
- reader/writer FAIL PROMPTLY (no TOCTOU gap — the exclusion is the DB's own
12073
- lock, which the advisory flocks do not provide against dashboard readers).
12074
- Exit 0 on success, 3 when the DB is in use."""
12901
+ reader/writer fail rather than hang (no TOCTOU gap — the exclusion is the
12902
+ DB's own lock, which the advisory flocks do not provide against dashboard
12903
+ readers). On a WAL store (#901 W9, Q15) the exclusive acquisition comes
12904
+ first and is retried until ``deadline`` while other processes' idle
12905
+ checkpoint keepers yield to the caller's request; the checkpoint and the
12906
+ VACUUM then run under that one hold. Exit 0 on success, 3 when the DB is
12907
+ in use."""
12075
12908
  conn = sqlite3.connect(f"file:{path}?mode=rw", uri=True)
12076
12909
  try:
12077
12910
  conn.execute(f"PRAGMA busy_timeout={_VACUUM_BUSY_TIMEOUT_MS}")
@@ -12080,6 +12913,7 @@ def _run_vacuum_exclusive(path, label: str) -> int:
12080
12913
  try:
12081
12914
  conn.execute("PRAGMA locking_mode=EXCLUSIVE")
12082
12915
  if label != "stats.db":
12916
+ _acquire_vacuum_exclusion(conn, deadline)
12083
12917
  conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
12084
12918
  conn.execute("VACUUM")
12085
12919
  except sqlite3.OperationalError as exc:
@@ -12189,7 +13023,13 @@ def _vacuum_one_db(path, label: str, provider_locked: bool) -> int:
12189
13023
  "before retrying VACUUM."
12190
13024
  )
12191
13025
  return 3
12192
- return _run_vacuum_exclusive(path, label)
13026
+ # stats.db has no checkpoint keeper: one attempt, as before.
13027
+ return _run_vacuum_exclusive(path, label)
13028
+ # #901 W9 (Q15): with both locks held, ask the other processes'
13029
+ # idle checkpoint keepers to close, close this process's own, and
13030
+ # retry only the exclusive acquisition within the drain grace.
13031
+ with _keeper_yield_request(path) as deadline:
13032
+ return _run_vacuum_exclusive(path, label, deadline=deadline)
12193
13033
  finally:
12194
13034
  for fh in held:
12195
13035
  try:
@@ -12212,9 +13052,12 @@ def cmd_db_vacuum(args: argparse.Namespace) -> int:
12212
13052
 
12213
13053
  NEVER automatic. Holds the maintenance flock + (for cache.db) the provider
12214
13054
  flocks, then runs a checkpoint + VACUUM under a real SQLite EXCLUSIVE lock so
12215
- a concurrent dashboard reader fails promptly instead of racing. Refuses when
12216
- free disk is below ~2x the file + WAL. Exit 0 on success, 3 when a target is
12217
- in use or disk is short."""
13055
+ a concurrent dashboard reader makes it refuse instead of racing. For
13056
+ cache.db and conversations.db it first asks other processes' idle
13057
+ checkpoint keepers to close (#901 W9, Q15) and retries the exclusive
13058
+ acquisition for at most 20 s; a handle still open then gets the same
13059
+ refusal. Refuses when free disk is below ~2x the file + WAL. Exit 0 on
13060
+ success, 3 when a target is in use or disk is short."""
12218
13061
  which = getattr(args, "db", "cache")
12219
13062
  targets = []
12220
13063
  if which in ("cache", "all"):