cctally 1.116.1 → 1.117.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +57 -3
  4. package/bin/_cctally_cache.py +5720 -885
  5. package/bin/_cctally_claude_attribution.py +997 -0
  6. package/bin/_cctally_completion.py +101 -28
  7. package/bin/_cctally_core.py +235 -12
  8. package/bin/_cctally_dashboard.py +263 -17
  9. package/bin/_cctally_dashboard_conversation.py +107 -21
  10. package/bin/_cctally_dashboard_envelope.py +44 -0
  11. package/bin/_cctally_db.py +928 -80
  12. package/bin/_cctally_diagnosis_sources.py +78 -13
  13. package/bin/_cctally_doctor.py +211 -86
  14. package/bin/_cctally_five_hour.py +446 -22
  15. package/bin/_cctally_forecast.py +302 -6
  16. package/bin/_cctally_journal.py +2418 -158
  17. package/bin/_cctally_milestone_history.py +379 -91
  18. package/bin/_cctally_oauth_samples.py +185 -7
  19. package/bin/_cctally_parser.py +38 -0
  20. package/bin/_cctally_project.py +34 -8
  21. package/bin/_cctally_quarantine.py +821 -343
  22. package/bin/_cctally_quota.py +69 -4
  23. package/bin/_cctally_quota_calibration.py +677 -3
  24. package/bin/_cctally_quota_model.py +927 -127
  25. package/bin/_cctally_record.py +922 -445
  26. package/bin/_cctally_rederive.py +119 -15
  27. package/bin/_cctally_statusline.py +1048 -176
  28. package/bin/_cctally_store.py +544 -110
  29. package/bin/_cctally_tui.py +455 -42
  30. package/bin/_cctally_update.py +77 -14
  31. package/bin/_lib_account_protocol.py +227 -19
  32. package/bin/_lib_claude_attribution.py +421 -0
  33. package/bin/_lib_clone_copy.py +30 -0
  34. package/bin/_lib_codex_conversation_query.py +45 -9
  35. package/bin/_lib_codex_conversation_watch.py +72 -13
  36. package/bin/_lib_codex_rollout_container.py +234 -19
  37. package/bin/_lib_codex_worker_lifecycle.py +374 -0
  38. package/bin/_lib_credit.py +22 -13
  39. package/bin/_lib_dashboard_sources.py +257 -18
  40. package/bin/_lib_detector_access_registry.py +1335 -166
  41. package/bin/_lib_doctor.py +276 -92
  42. package/bin/_lib_ingest_frontier.py +269 -36
  43. package/bin/_lib_journal.py +273 -22
  44. package/bin/_lib_jsonl.py +12 -0
  45. package/bin/_lib_meter_rate_change.py +72 -0
  46. package/bin/_lib_outcome_quarantine.py +917 -554
  47. package/bin/_lib_provider_activity.py +24 -6
  48. package/bin/_lib_quota_calibration.py +57 -3
  49. package/bin/_lib_quota_copy.py +7 -2
  50. package/bin/_lib_quota_lineage.py +613 -0
  51. package/bin/_lib_quota_model.py +464 -33
  52. package/bin/_lib_rate_change_delivery.py +16 -0
  53. package/bin/_lib_reclaim_planner.py +6 -4
  54. package/bin/_lib_rederive.py +2046 -21
  55. package/bin/_lib_selector_state.py +131 -3
  56. package/bin/_lib_snapshot_cache.py +8 -0
  57. package/bin/_lib_stale_proof.py +113 -4
  58. package/bin/_lib_statusline.py +72 -3
  59. package/bin/_lib_statusline_candidates.py +246 -42
  60. package/bin/_lib_tick_stats.py +10 -5
  61. package/bin/_lib_write_io.py +419 -6
  62. package/bin/cctally +28 -11
  63. package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
  64. package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
  65. package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
  66. package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
  67. package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
  68. package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
  69. package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
  70. package/dashboard/static/assets/index-D34eYhAc.css +1 -0
  71. package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
  72. package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
  73. package/dashboard/static/dashboard.html +3 -3
  74. package/package.json +5 -1
  75. package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
  76. package/dashboard/static/assets/index-BKITk1CT.css +0 -1
@@ -64,6 +64,7 @@ import contextlib
64
64
  import contextvars
65
65
  import datetime as dt
66
66
  import enum
67
+ import errno
67
68
  import fcntl
68
69
  import json
69
70
  import os
@@ -71,6 +72,7 @@ import pathlib
71
72
  import re
72
73
  import shutil
73
74
  import sqlite3
75
+ import stat
74
76
  import subprocess
75
77
  import sys
76
78
  import tempfile
@@ -1068,6 +1070,8 @@ def write_corruption_forensics(
1068
1070
  trigger_origin: "str | None" = None,
1069
1071
  trigger_exception: "BaseException | None" = None,
1070
1072
  return_result: bool = False,
1073
+ probe_immutable: bool = False,
1074
+ before_publish: "Callable[[], None] | None" = None,
1071
1075
  ) -> "pathlib.Path | None | CorruptionForensicsResult":
1072
1076
  """Write the #336-defense-2 corruption forensics bundle FIRST — before any
1073
1077
  quarantine/rebuild disturbs the evidence (spec §6.3). Captures the file family
@@ -1083,6 +1087,15 @@ def write_corruption_forensics(
1083
1087
  isolated family snapshot while the bundle still identifies the live
1084
1088
  ``db_path``. Probe/write unavailability is always unconfirmed. Confirmed
1085
1089
  bundles are atomically replaced and fsynced before this function returns.
1090
+
1091
+ #1000: ``probe_immutable=True`` opens the integrity connection on
1092
+ ``probe_db_path`` as ``mode=ro&immutable=1`` through an escaped absolute
1093
+ URI, for a caller that checks a live main file in place: a plain
1094
+ ``mode=ro`` open of a WAL-mode store creates ``-wal``/``-shm`` beside it.
1095
+ ``before_publish``, when given, runs after the integrity probe and the
1096
+ damage scan and immediately before the bundle is published; an exception
1097
+ it raises propagates (it is outside every handler here) and nothing is
1098
+ published for that attempt.
1086
1099
  """
1087
1100
  if (trigger_origin is None) != (trigger_exception is None):
1088
1101
  raise ValueError(
@@ -1157,8 +1170,15 @@ def write_corruption_forensics(
1157
1170
  }
1158
1171
  except OSError:
1159
1172
  bundle["family"][p.name] = None
1173
+ if probe_immutable:
1174
+ probe_uri = (
1175
+ pathlib.Path(os.path.abspath(probe_db_path)).as_uri()
1176
+ + "?mode=ro&immutable=1"
1177
+ )
1178
+ else:
1179
+ probe_uri = f"file:{probe_db_path}?mode=ro"
1160
1180
  try:
1161
- c = sqlite3.connect(f"file:{probe_db_path}?mode=ro", uri=True)
1181
+ c = sqlite3.connect(probe_uri, uri=True)
1162
1182
  try:
1163
1183
  c.execute("PRAGMA busy_timeout=2000")
1164
1184
  bundle["integrityCheck"] = [
@@ -1214,6 +1234,8 @@ def write_corruption_forensics(
1214
1234
  bundle["holders"].append(line.strip())
1215
1235
  except Exception as exc:
1216
1236
  bundle["holders"] = [f"error: {exc}"]
1237
+ if before_publish is not None:
1238
+ before_publish()
1217
1239
  try:
1218
1240
  _atomic_write_private_json(out, bundle)
1219
1241
  except OSError:
@@ -4380,6 +4402,210 @@ def _apply_codex_accounting_change_ledger(conn: sqlite3.Connection) -> None:
4380
4402
  conn.execute(statement)
4381
4403
 
4382
4404
 
4405
+ # ── #927 (#981 Part B): the retained-identity inventory (spec §4.2 B1) ───────
4406
+ #
4407
+ # The pre-walk prune inventory used to read every retained Codex source
4408
+ # identity from four history-sized families on every hook (13.2-13.5 s cold on
4409
+ # the measured store). This table holds each distinct ``(source_path,
4410
+ # source_root_key)`` those families have carried, maintained by INSERT and
4411
+ # identity-UPDATE triggers in the same transaction as the write. There are no
4412
+ # DELETE triggers: deletion owners reconcile the table themselves, and every
4413
+ # other deletion leaves a SUPERSET, which the planner confirms with indexed
4414
+ # existence probes before an identity becomes protection evidence.
4415
+ #
4416
+ # A NULL root is distinct from every non-NULL root, and ``''`` from NULL, so
4417
+ # the uniqueness is an expression index rather than a plain UNIQUE. Each
4418
+ # trigger body inserts only when the identity is absent: an outer
4419
+ # statement's conflict policy propagates into trigger-body INSERTs (the #582
4420
+ # ledger note above), so the ``NOT EXISTS`` keeps a conflict from ever being
4421
+ # raised whatever that policy is.
4422
+ #
4423
+ # ``cache_meta.codex_identity_inventory`` is ``ready`` or
4424
+ # ``bootstrap:<family>:<last_rowid>``; any other value reads as
4425
+ # ``bootstrap:files:0``. Only ``ready`` lets the planner read the table.
4426
+
4427
+ CODEX_IDENTITY_INVENTORY_KEY = "codex_identity_inventory"
4428
+ CODEX_IDENTITY_INVENTORY_READY = "ready"
4429
+ #: The bootstrap's fixed family order, each with its rowid source query.
4430
+ CODEX_IDENTITY_INVENTORY_FAMILIES: "tuple[tuple[str, str, str, str], ...]" = (
4431
+ # (state token, table, path column, extra WHERE)
4432
+ ("files", "codex_session_files", "path", ""),
4433
+ ("entries", "codex_session_entries", "source_path", ""),
4434
+ ("quota", "quota_window_snapshots", "source_path", " AND source = 'codex'"),
4435
+ ("threads", "codex_conversation_threads", "source_path", ""),
4436
+ )
4437
+
4438
+ _CODEX_RETAINED_IDENTITY_OBJECTS = (
4439
+ ("table", "codex_retained_identities"),
4440
+ ("index", "idx_codex_retained_identities_key"),
4441
+ ("trigger", "trg_codex_identity_files_ins"),
4442
+ ("trigger", "trg_codex_identity_files_upd"),
4443
+ ("trigger", "trg_codex_identity_entries_ins"),
4444
+ ("trigger", "trg_codex_identity_entries_upd"),
4445
+ ("trigger", "trg_codex_identity_quota_ins"),
4446
+ ("trigger", "trg_codex_identity_quota_upd"),
4447
+ ("trigger", "trg_codex_identity_threads_ins"),
4448
+ ("trigger", "trg_codex_identity_threads_upd"),
4449
+ )
4450
+
4451
+
4452
+ def _codex_retained_identity_ddl() -> "tuple[str, ...]":
4453
+ def add(path: str) -> str:
4454
+ return f"""
4455
+ INSERT OR IGNORE INTO codex_retained_identities
4456
+ (source_path, source_root_key)
4457
+ SELECT NEW.{path}, NEW.source_root_key
4458
+ WHERE NOT EXISTS (
4459
+ SELECT 1 FROM codex_retained_identities
4460
+ WHERE source_path = NEW.{path}
4461
+ AND IFNULL(source_root_key, char(0))
4462
+ = IFNULL(NEW.source_root_key, char(0))
4463
+ );
4464
+ """
4465
+ files, entries = add("path"), add("source_path")
4466
+ return (
4467
+ """
4468
+ CREATE TABLE IF NOT EXISTS codex_retained_identities (
4469
+ source_path TEXT NOT NULL,
4470
+ source_root_key TEXT
4471
+ )
4472
+ """,
4473
+ "CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_retained_identities_key "
4474
+ "ON codex_retained_identities"
4475
+ "(source_path, IFNULL(source_root_key, char(0)))",
4476
+ "DROP TRIGGER IF EXISTS trg_codex_identity_files_ins",
4477
+ f"""
4478
+ CREATE TRIGGER trg_codex_identity_files_ins
4479
+ AFTER INSERT ON codex_session_files
4480
+ BEGIN {files} END
4481
+ """,
4482
+ "DROP TRIGGER IF EXISTS trg_codex_identity_files_upd",
4483
+ f"""
4484
+ CREATE TRIGGER trg_codex_identity_files_upd
4485
+ AFTER UPDATE OF path, source_root_key ON codex_session_files
4486
+ BEGIN {files} END
4487
+ """,
4488
+ "DROP TRIGGER IF EXISTS trg_codex_identity_entries_ins",
4489
+ f"""
4490
+ CREATE TRIGGER trg_codex_identity_entries_ins
4491
+ AFTER INSERT ON codex_session_entries
4492
+ BEGIN {entries} END
4493
+ """,
4494
+ "DROP TRIGGER IF EXISTS trg_codex_identity_entries_upd",
4495
+ f"""
4496
+ CREATE TRIGGER trg_codex_identity_entries_upd
4497
+ AFTER UPDATE OF source_path, source_root_key ON codex_session_entries
4498
+ BEGIN {entries} END
4499
+ """,
4500
+ "DROP TRIGGER IF EXISTS trg_codex_identity_quota_ins",
4501
+ f"""
4502
+ CREATE TRIGGER trg_codex_identity_quota_ins
4503
+ AFTER INSERT ON quota_window_snapshots
4504
+ WHEN NEW.source = 'codex'
4505
+ BEGIN {entries} END
4506
+ """,
4507
+ "DROP TRIGGER IF EXISTS trg_codex_identity_quota_upd",
4508
+ f"""
4509
+ CREATE TRIGGER trg_codex_identity_quota_upd
4510
+ AFTER UPDATE OF source, source_path, source_root_key
4511
+ ON quota_window_snapshots
4512
+ WHEN NEW.source = 'codex'
4513
+ BEGIN {entries} END
4514
+ """,
4515
+ "DROP TRIGGER IF EXISTS trg_codex_identity_threads_ins",
4516
+ f"""
4517
+ CREATE TRIGGER trg_codex_identity_threads_ins
4518
+ AFTER INSERT ON codex_conversation_threads
4519
+ BEGIN {entries} END
4520
+ """,
4521
+ "DROP TRIGGER IF EXISTS trg_codex_identity_threads_upd",
4522
+ f"""
4523
+ CREATE TRIGGER trg_codex_identity_threads_upd
4524
+ AFTER UPDATE OF source_path, source_root_key
4525
+ ON codex_conversation_threads
4526
+ BEGIN {entries} END
4527
+ """,
4528
+ )
4529
+
4530
+
4531
+ def _codex_identity_families_empty(conn: sqlite3.Connection) -> bool:
4532
+ """True when none of the four families holds a Codex row (O(1) each)."""
4533
+ return not any(
4534
+ conn.execute(
4535
+ f"SELECT 1 FROM {table} WHERE 1{extra} LIMIT 1").fetchone()
4536
+ for _token, table, _path, extra in CODEX_IDENTITY_INVENTORY_FAMILIES
4537
+ )
4538
+
4539
+
4540
+ def _apply_codex_retained_identity_inventory(conn: sqlite3.Connection) -> None:
4541
+ """Install the retained-identity inventory on every delivery path (B1).
4542
+
4543
+ Shared by ``_apply_cache_schema`` and migration 051 so fresh and upgraded
4544
+ stores converge. Atomic like the #901 latest summary: the table, its
4545
+ index, the eight triggers and the state key commit together, as a
4546
+ savepoint inside the caller's transaction or as an IMMEDIATE transaction
4547
+ of their own. Triggers come first, so no write can land between the
4548
+ state decision and the maintenance that keeps it true.
4549
+
4550
+ The state key is decided only when any of the ten objects was missing
4551
+ before this call (a store carrying all ten has been maintained since they
4552
+ were created, so its recorded state stands): ``ready`` when the four
4553
+ families hold no Codex row, else ``bootstrap:files:0`` for the off-hook
4554
+ backfill. A history scan never runs here. Guarded on the identity columns,
4555
+ which a legacy-shape cache whose schema apply returned early can lack.
4556
+ """
4557
+ def _columns(table: str) -> "set[str]":
4558
+ return {
4559
+ str(row[0]) for row in conn.execute(
4560
+ "SELECT name FROM pragma_table_info(?)", (table,))
4561
+ }
4562
+
4563
+ if not (
4564
+ {"path", "source_root_key"} <= _columns("codex_session_files")
4565
+ and {"source_path", "source_root_key"} <= _columns(
4566
+ "codex_session_entries")
4567
+ and {"source", "source_path", "source_root_key"} <= _columns(
4568
+ "quota_window_snapshots")
4569
+ and {"source_path", "source_root_key"} <= _columns(
4570
+ "codex_conversation_threads")
4571
+ ):
4572
+ return
4573
+ own_transaction = not conn.in_transaction
4574
+ conn.execute(
4575
+ "BEGIN IMMEDIATE" if own_transaction
4576
+ else "SAVEPOINT codex_retained_identity_apply")
4577
+ try:
4578
+ present = {
4579
+ (str(row[0]), str(row[1])) for row in conn.execute(
4580
+ "SELECT type, name FROM sqlite_master "
4581
+ "WHERE type IN ('table', 'index', 'trigger')")
4582
+ }
4583
+ complete = all(
4584
+ item in present for item in _CODEX_RETAINED_IDENTITY_OBJECTS)
4585
+ for statement in _codex_retained_identity_ddl():
4586
+ conn.execute(statement)
4587
+ if not complete:
4588
+ state = (
4589
+ CODEX_IDENTITY_INVENTORY_READY
4590
+ if _codex_identity_families_empty(conn)
4591
+ else "bootstrap:files:0")
4592
+ conn.execute(
4593
+ "INSERT INTO cache_meta(key, value) VALUES (?, ?) "
4594
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
4595
+ (CODEX_IDENTITY_INVENTORY_KEY, state))
4596
+ except BaseException:
4597
+ if own_transaction:
4598
+ conn.rollback()
4599
+ else:
4600
+ conn.execute("ROLLBACK TO codex_retained_identity_apply")
4601
+ conn.execute("RELEASE codex_retained_identity_apply")
4602
+ raise
4603
+ if own_transaction:
4604
+ conn.commit()
4605
+ else:
4606
+ conn.execute("RELEASE codex_retained_identity_apply")
4607
+
4608
+
4383
4609
  # ── #901 W1: the maintained raw-partition maxima (spec §5.1, T1) ─────────────
4384
4610
  #
4385
4611
  # `load_codex_quota_observations(latest_per_identity=True)` computed
@@ -4808,6 +5034,46 @@ _CODEX_CONTAINER_CURSOR_COLUMNS: "tuple[tuple[str, str], ...]" = (
4808
5034
  _CODEX_CONTAINER_CACHE_ONLY_COLUMNS: "tuple[tuple[str, str], ...]" = (
4809
5035
  ("container_resume_strikes", "INTEGER NOT NULL DEFAULT 0"),
4810
5036
  )
5037
+ #: #944 spec 5.1/5.4: the derived tables of write-time Claude cost
5038
+ #: attribution. Both are journal-backed and disposable: the decisions replay
5039
+ #: from `claude_cost_account` ops, the timeline from admitted Claude
5040
+ #: `record-usage` observations.
5041
+ _CLAUDE_ATTRIBUTION_CACHE_TABLES = (
5042
+ "claude_account_timeline", "claude_cost_decisions",
5043
+ )
5044
+ _CLAUDE_ATTRIBUTION_CACHE_DDL = (
5045
+ "CREATE TABLE IF NOT EXISTS claude_account_timeline ("
5046
+ " captured_at_utc TEXT NOT NULL,"
5047
+ " precision_us INTEGER NOT NULL,"
5048
+ " account_key TEXT NULL,"
5049
+ " journal_segment TEXT NOT NULL,"
5050
+ " journal_offset INTEGER NOT NULL)",
5051
+ "CREATE INDEX IF NOT EXISTS idx_claude_account_timeline_captured "
5052
+ "ON claude_account_timeline(captured_at_utc)",
5053
+ "CREATE TABLE IF NOT EXISTS claude_cost_decisions ("
5054
+ " decision_key TEXT PRIMARY KEY,"
5055
+ " form TEXT NOT NULL,"
5056
+ " msg_id TEXT,"
5057
+ " req_id TEXT,"
5058
+ " loc TEXT,"
5059
+ " byte_offset INTEGER,"
5060
+ " digest TEXT,"
5061
+ " account_key TEXT NULL,"
5062
+ " basis TEXT NOT NULL,"
5063
+ " reason TEXT NULL,"
5064
+ " rule INTEGER NOT NULL,"
5065
+ " decided_at_utc TEXT NOT NULL,"
5066
+ " record_ts_utc TEXT NULL,"
5067
+ " projection_json TEXT NULL,"
5068
+ " journal_segment TEXT NOT NULL,"
5069
+ " journal_offset INTEGER NOT NULL)",
5070
+ "CREATE INDEX IF NOT EXISTS idx_claude_cost_decisions_decided "
5071
+ "ON claude_cost_decisions(decided_at_utc)",
5072
+ "CREATE INDEX IF NOT EXISTS idx_claude_cost_decisions_loc "
5073
+ "ON claude_cost_decisions(loc, byte_offset)",
5074
+ )
5075
+
5076
+
4811
5077
  _CODEX_968_CACHE_TABLES = (
4812
5078
  "codex_prune_debt", "codex_barrier_membership", "codex_barrier_targets",
4813
5079
  "codex_barrier_population", "codex_barrier_certificates",
@@ -5209,6 +5475,28 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
5209
5475
  "index", "idx_codex_window_attributions_read_order",
5210
5476
  "_apply_codex_window_attribution_read_index",
5211
5477
  "047_spill_free_read_paths"),
5478
+ # #944 spec 5.7: migration 050's handler calls the shared ensure helper.
5479
+ *(SchemaDeliveryObject(
5480
+ "table", table, "_apply_claude_attribution_schema",
5481
+ "050_preserve_claude_cost_attribution")
5482
+ for table in _CLAUDE_ATTRIBUTION_CACHE_TABLES),
5483
+ SchemaDeliveryObject(
5484
+ "column", "session_entries.attribution_decision_key",
5485
+ "_apply_claude_attribution_schema",
5486
+ "050_preserve_claude_cost_attribution"),
5487
+ *(SchemaDeliveryObject(
5488
+ "index", index, "_apply_claude_attribution_schema",
5489
+ "050_preserve_claude_cost_attribution")
5490
+ for index in ("idx_claude_account_timeline_captured",
5491
+ "idx_claude_cost_decisions_decided",
5492
+ "idx_claude_cost_decisions_loc",
5493
+ "idx_entries_attr_unlinked",
5494
+ "idx_entries_attr_nonkeyed")),
5495
+ # #927 spec §4.2 B1: migration 051's handler calls the shared helper.
5496
+ *(SchemaDeliveryObject(
5497
+ kind, name, "_apply_codex_retained_identity_inventory",
5498
+ "051_codex_retained_identities")
5499
+ for kind, name in _CODEX_RETAINED_IDENTITY_OBJECTS),
5212
5500
  )
5213
5501
 
5214
5502
 
@@ -5251,6 +5539,51 @@ def _apply_codex_container_cache_schema(conn: sqlite3.Connection) -> None:
5251
5539
  conn.execute(statement)
5252
5540
 
5253
5541
 
5542
+ def _apply_claude_attribution_schema(conn: sqlite3.Connection) -> None:
5543
+ """#944 spec 5.1/5.4: deliver the write-time Claude cost attribution
5544
+ schema -- the two derived tables, the ``session_entries`` link column and
5545
+ its two partial indexes.
5546
+
5547
+ Shared by ``_apply_cache_schema`` and the migration that delivers it to an
5548
+ already-current store, so both shapes converge. The link column goes
5549
+ through ``add_column_if_missing`` only, never the CREATE TABLE body
5550
+ (ALTER TABLE ADD COLUMN appends, so a column in both places sits at a
5551
+ different ordinal on a fresh store than on a migrated one -- the #195
5552
+ hazard). Idempotent.
5553
+ """
5554
+ for statement in _CLAUDE_ATTRIBUTION_CACHE_DDL:
5555
+ conn.execute(statement)
5556
+ if _table_exists(conn, "session_entries"):
5557
+ # Literal column name keeps the schema-delivery AST guard complete.
5558
+ add_column_if_missing(
5559
+ conn, "session_entries", "attribution_decision_key", "TEXT")
5560
+ conn.execute(
5561
+ "CREATE INDEX IF NOT EXISTS idx_entries_attr_unlinked "
5562
+ "ON session_entries(id) WHERE attribution_decision_key IS NULL")
5563
+ conn.execute(
5564
+ "CREATE INDEX IF NOT EXISTS idx_entries_attr_nonkeyed "
5565
+ "ON session_entries(attribution_decision_key) "
5566
+ "WHERE msg_id IS NULL OR req_id IS NULL")
5567
+
5568
+
5569
+ def claude_attribution_schema_present(conn: sqlite3.Connection) -> bool:
5570
+ """Whether this store carries the #944 attribution schema (both tables
5571
+ and the link column). A direct handler call on a historical
5572
+ ``pre.sqlite`` lacks it; every production store has it, because schema
5573
+ delivery precedes every handler (spec 5.6)."""
5574
+ try:
5575
+ tables = {row[0] for row in conn.execute(
5576
+ "SELECT name FROM sqlite_master WHERE type='table' AND name IN "
5577
+ "('claude_account_timeline', 'claude_cost_decisions')")}
5578
+ if len(tables) != 2:
5579
+ return False
5580
+ columns = {row[1] for row in conn.execute(
5581
+ "PRAGMA table_info(session_entries)")}
5582
+ except sqlite3.Error:
5583
+ return False
5584
+ return "attribution_decision_key" in columns
5585
+
5586
+
5254
5587
  def _apply_codex_conversation_container_schema(conn: sqlite3.Connection) -> None:
5255
5588
  """#968 spec 5.3: the conversations twin of the container evidence.
5256
5589
 
@@ -5924,6 +6257,10 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
5924
6257
  "ON session_entries(source_path, line_offset)")
5925
6258
  except sqlite3.IntegrityError:
5926
6259
  pass
6260
+ # #944: write-time Claude cost attribution (derived tables, the link
6261
+ # column and its partial indexes). After the session_entries column adds
6262
+ # and BEFORE the legacy-FTS early return, like the indexes above.
6263
+ _apply_claude_attribution_schema(conn)
5927
6264
  # Existing-DB guard for the skill-content fold link (cctally-dev
5928
6265
  # skill-content-nesting): the message-level sourceToolUseID. Idempotent
5929
6266
  # column-add (no marker, no version); cache migration 006 then re-ingests
@@ -5995,6 +6332,8 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
5995
6332
  _apply_codex_quota_unresolved_model_index(conn)
5996
6333
  _apply_codex_quota_change_ledger(conn)
5997
6334
  _apply_codex_accounting_change_ledger(conn)
6335
+ # #927: after cache_meta and the four families exist (spec §4.2 B1).
6336
+ _apply_codex_retained_identity_inventory(conn)
5998
6337
  # #901 (spec §5.1-§5.3): the latest-capture summary, the general loader's
5999
6338
  # ordering index, the W2 lookup indexes, the pricing model indexes and the
6000
6339
  # window-attribution read order (Amendment 1 item 1). BEFORE the
@@ -6194,6 +6533,13 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
6194
6533
  -- quota_window_snapshots, which was dropped above together with its
6195
6534
  -- triggers and seek index. Drop the table as well.
6196
6535
  DROP TABLE IF EXISTS codex_quota_partition_latest;
6536
+ -- #944: the Claude cost attribution tables are cache.db accounting
6537
+ -- state; conversation code reads them through the cache_db attachment.
6538
+ DROP TABLE IF EXISTS claude_account_timeline;
6539
+ DROP TABLE IF EXISTS claude_cost_decisions;
6540
+ -- #927: the retained-identity inventory is cache.db prune state. Its
6541
+ -- triggers went with the four family tables dropped above.
6542
+ DROP TABLE IF EXISTS codex_retained_identities;
6197
6543
 
6198
6544
  CREATE TABLE IF NOT EXISTS conversation_source_files (
6199
6545
  path TEXT PRIMARY KEY,
@@ -8280,6 +8626,11 @@ def _001_dedup_highest_wins_locked(conn: sqlite3.Connection) -> None:
8280
8626
  "corrected dedup (one-time; may take 10-30s depending on "
8281
8627
  "JSONL volume)..."
8282
8628
  )
8629
+ # #944 spec 5.6: no row is deleted before it is linked. Every unlinked
8630
+ # row's stamp (NULL included, whatever the population: a pending 029 is no
8631
+ # proof that NULLs predate account stamping) is aligned or exported first;
8632
+ # an unwritable journal defers 001 with nothing wiped.
8633
+ _link_claude_rows_before_001_wipe(conn)
8283
8634
  # D3 — BEGIN IMMEDIATE so the destructive DELETEs are race-guarded,
8284
8635
  # not just the marker insert. The dispatcher snapshots the applied
8285
8636
  # set ONCE before its registry walk (``_run_pending_migrations``),
@@ -8342,6 +8693,48 @@ def _001_dedup_highest_wins_locked(conn: sqlite3.Connection) -> None:
8342
8693
  raise
8343
8694
 
8344
8695
 
8696
+ #: #944 spec 5.7: unlinked rows one migration-050 invocation links (and the
8697
+ #: round size migration 001's preserve-before-wipe commits per round).
8698
+ _CLAUDE_PRESERVATION_CHUNK_ROWS = 50_000
8699
+
8700
+
8701
+ def _link_claude_rows_before_001_wipe(conn: sqlite3.Connection) -> None:
8702
+ """Link every unlinked ``session_entries`` row before cache 001 wipes the
8703
+ table (#944 spec 5.6): replay pending decisions, then align or export in
8704
+ committed rounds. A store without the attribution schema (a direct handler
8705
+ call on a historical ``pre.sqlite``) holds no decisions and behaves
8706
+ exactly as before. ``MigrationGateNotMet`` when the journal cannot be read
8707
+ or written, so 001 stays pending with nothing wiped."""
8708
+ if not claude_attribution_schema_present(conn):
8709
+ return
8710
+ if conn.execute(
8711
+ "SELECT 1 FROM schema_migrations WHERE name = ? LIMIT 1",
8712
+ ("001_dedup_highest_wins",),
8713
+ ).fetchone() is not None:
8714
+ return # a concurrent opener already wiped; nothing left to link
8715
+ import _cctally_claude_attribution as G
8716
+ try:
8717
+ hw = G.rehydrate_claude_attribution(conn, authoritative=False)
8718
+ if hw is None:
8719
+ raise G.AttributionJournalUnavailable(
8720
+ "the journal could not be read")
8721
+ conn.commit()
8722
+ after = 0
8723
+ while True:
8724
+ result = G.preserve_unlinked_rows(
8725
+ conn, journal_hw=hw, after_id=after,
8726
+ limit=_CLAUDE_PRESERVATION_CHUNK_ROWS)
8727
+ conn.commit()
8728
+ if not result.remaining:
8729
+ return
8730
+ after = result.last_id
8731
+ except G.AttributionJournalUnavailable as exc:
8732
+ conn.rollback()
8733
+ raise MigrationGateNotMet(
8734
+ f"Claude cost attribution could not be made durable before the "
8735
+ f"cache 001 wipe ({exc}); deferring") from exc
8736
+
8737
+
8345
8738
  # === Region 7c: Cache migration 002_conversation_messages_backfill ===
8346
8739
 
8347
8740
  @cache_migration("002_conversation_messages_backfill")
@@ -9339,16 +9732,24 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
9339
9732
  resolved (not pending) outcome → no-op (leave NULL). Idempotent: only touches
9340
9733
  ``account_key IS NULL`` rows. Central stamp via the dispatcher (#140); the
9341
9734
  handler does NOT self-stamp.
9735
+
9736
+ #944 spec 5.6: a row linked to a durable Claude cost decision (a decided
9737
+ NULL included) is never overwritten, whatever the skip/unskip order. A
9738
+ store without the attribution schema (a direct call on a historical
9739
+ ``pre.sqlite``) has no links, so the predicate is omitted there.
9342
9740
  """
9343
9741
  import _cctally_journal as _jr
9344
9742
  import _lib_accounts as _acc
9345
9743
 
9744
+ unlinked = (" AND attribution_decision_key IS NULL"
9745
+ if claude_attribution_schema_present(conn) else "")
9346
9746
  op_value = _accounts_cutover_value_or_defer("Claude account backfill")
9347
9747
  if op_value is None:
9348
9748
  # Cutover hasn't appended its op yet. Defer only if there is legacy Claude
9349
9749
  # history to attribute; a fresh install (no such rows) proceeds as a no-op.
9350
9750
  pending = conn.execute(
9351
- "SELECT 1 FROM session_entries WHERE account_key IS NULL LIMIT 1"
9751
+ "SELECT 1 FROM session_entries WHERE account_key IS NULL"
9752
+ + unlinked + " LIMIT 1"
9352
9753
  ).fetchone()
9353
9754
  if pending is not None:
9354
9755
  raise MigrationGateNotMet(
@@ -9361,7 +9762,8 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
9361
9762
  conn.execute("BEGIN IMMEDIATE")
9362
9763
  try:
9363
9764
  conn.execute(
9364
- "UPDATE session_entries SET account_key=? WHERE account_key IS NULL",
9765
+ "UPDATE session_entries SET account_key=? WHERE account_key IS NULL"
9766
+ + unlinked,
9365
9767
  (op_value,),
9366
9768
  )
9367
9769
  conn.execute(
@@ -10268,6 +10670,144 @@ def _049_codex_rollout_containers(conn: sqlite3.Connection) -> None:
10268
10670
  conn.commit()
10269
10671
 
10270
10672
 
10673
+ @cache_migration("050_preserve_claude_cost_attribution")
10674
+ def _050_preserve_claude_cost_attribution(conn: sqlite3.Connection) -> None:
10675
+ """#944 spec 5.7 (U1): make every existing Claude cost stamp durable.
10676
+
10677
+ Delivers the attribution schema (the shared helper every fresh store
10678
+ gets through ``_apply_cache_schema``), then links up to
10679
+ ``_CLAUDE_PRESERVATION_CHUNK_ROWS`` unlinked ``session_entries`` rows per
10680
+ invocation, in ``id`` order past the recorded cursor: a row whose
10681
+ identity already holds a decision is aligned to it, every other row's
10682
+ CURRENT stamp (NULL included, orphan sources included) is exported as a
10683
+ ``preserved_history`` decision -- keyed form, or the legacy projection
10684
+ form for a row lacking an id. Never consults the kernel and never reads
10685
+ a transcript.
10686
+
10687
+ Gate: migration 029 applied or skipped, because 029 backfills NULL stamps
10688
+ U1 then preserves as they stand. Each chunk appends (fsync) and then, in
10689
+ one transaction, inserts the decisions, sets the links and records
10690
+ ``claude_cost_preservation``; while an unlinked row remains past the
10691
+ cursor the handler raises ``MigrationGateNotMet`` so the migration stays
10692
+ pending with its committed chunks kept. An unwritable journal records
10693
+ ``lastError``/``lastErrorAt`` in a separate small transaction (doctor's
10694
+ stall WARN) and defers; nothing is wiped and nothing is stamped.
10695
+
10696
+ Takes the cache writer flock on a direct call (the dispatcher already
10697
+ holds it). Re-running is a no-op once every row is linked. NO
10698
+ self-stamp -- the dispatcher central-stamps on a clean return (#140).
10699
+ """
10700
+ _apply_claude_attribution_schema(conn)
10701
+ conn.commit()
10702
+ gate = conn.execute(
10703
+ "SELECT 1 FROM schema_migrations WHERE name = ? UNION ALL "
10704
+ "SELECT 1 FROM schema_migrations_skipped WHERE name = ? LIMIT 1",
10705
+ ("029_backfill_claude_account", "029_backfill_claude_account"),
10706
+ ).fetchone() if _table_exists(conn, "schema_migrations_skipped") else (
10707
+ conn.execute(
10708
+ "SELECT 1 FROM schema_migrations WHERE name = ? LIMIT 1",
10709
+ ("029_backfill_claude_account",)).fetchone())
10710
+ if gate is None:
10711
+ raise MigrationGateNotMet(
10712
+ "cache 029 is neither applied nor skipped; deferring the Claude "
10713
+ "cost attribution export until its NULL backfill has run")
10714
+ lock_path = (
10715
+ None
10716
+ if _CACHE_DISPATCH_WRITER_LOCK_HELD.get()
10717
+ else _cache_db_lock_path_for_conn(conn)
10718
+ )
10719
+ lock_fh = None
10720
+ if lock_path is not None:
10721
+ lock_fh = open(lock_path, "w")
10722
+ try:
10723
+ fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
10724
+ except BlockingIOError:
10725
+ lock_fh.close()
10726
+ raise MigrationGateNotMet(
10727
+ "cache.db.lock held by a concurrent sync_cache; deferring "
10728
+ "cache 050 Claude cost attribution export (#944)")
10729
+ try:
10730
+ _050_export_one_chunk(conn)
10731
+ finally:
10732
+ if lock_fh is not None:
10733
+ try:
10734
+ fcntl.flock(lock_fh, fcntl.LOCK_UN)
10735
+ except OSError:
10736
+ pass
10737
+ lock_fh.close()
10738
+
10739
+
10740
+ def _050_export_one_chunk(conn: sqlite3.Connection) -> None:
10741
+ import _cctally_claude_attribution as G
10742
+
10743
+ def unlinked_count() -> int:
10744
+ return conn.execute(
10745
+ "SELECT COUNT(*) FROM session_entries "
10746
+ "WHERE attribution_decision_key IS NULL").fetchone()[0]
10747
+
10748
+ progress = G.read_preservation(conn)
10749
+ if progress is None:
10750
+ progress = {"afterId": 0, "saved": 0, "total": unlinked_count(),
10751
+ "lastError": None, "lastErrorAt": None,
10752
+ "lastChunkAt": None}
10753
+ after = int(progress.get("afterId") or 0)
10754
+ try:
10755
+ hw = G.rehydrate_claude_attribution(conn, authoritative=False)
10756
+ if hw is None:
10757
+ raise G.AttributionJournalUnavailable(
10758
+ "the journal could not be read")
10759
+ conn.commit()
10760
+ result = G.preserve_unlinked_rows(
10761
+ conn, journal_hw=hw, after_id=after,
10762
+ limit=_CLAUDE_PRESERVATION_CHUNK_ROWS)
10763
+ except G.AttributionJournalUnavailable as exc:
10764
+ conn.rollback()
10765
+ if not exc.contended:
10766
+ progress["lastError"] = str(exc)
10767
+ progress["lastErrorAt"] = now_utc_iso()
10768
+ G.write_preservation(conn, progress)
10769
+ conn.commit()
10770
+ raise MigrationGateNotMet(
10771
+ f"Claude cost attribution export deferred: {exc}") from exc
10772
+ total = int(progress.get("total") or 0)
10773
+ progress.update({
10774
+ "afterId": max(after, result.last_id),
10775
+ # Aligned rows and rows ingest linked meanwhile count as saved.
10776
+ "saved": max(0, min(total, total - unlinked_count())),
10777
+ "lastError": None, "lastErrorAt": None,
10778
+ "lastChunkAt": now_utc_iso(),
10779
+ })
10780
+ G.write_preservation(conn, progress)
10781
+ conn.commit()
10782
+ if result.remaining:
10783
+ raise MigrationGateNotMet(
10784
+ f"Claude cost attribution export saved {progress['saved']} of "
10785
+ f"{total} rows; resuming on the next open")
10786
+
10787
+
10788
+ @cache_migration("051_codex_retained_identities")
10789
+ def _051_codex_retained_identities(conn: sqlite3.Connection) -> None:
10790
+ """#927 spec §4.2 B1: deliver the retained-identity inventory.
10791
+
10792
+ The schema apply that declares the table, its expression unique index and
10793
+ the eight INSERT/identity-UPDATE triggers is version-gated, so only a
10794
+ registered migration reaches an already-current store (the reason 046
10795
+ gives). The shared helper creates the triggers first and then records
10796
+ ``ready`` when the four families hold no Codex row, else
10797
+ ``bootstrap:files:0``: an existing history is backfilled off the hook by
10798
+ ``_codex-inventory-bootstrap`` (or inline by ``cache-sync``), never here,
10799
+ so an ordinary open pays no history scan.
10800
+
10801
+ Takes no provider flock: the helper's IMMEDIATE transaction makes the
10802
+ trigger creation and the state decision atomic against every writer.
10803
+ Re-running is a no-op once all ten objects exist (the recorded state, and
10804
+ any bootstrap progress, stand). NO self-stamp — the dispatcher
10805
+ central-stamps on a clean return (#140).
10806
+ """
10807
+ _apply_codex_retained_identity_inventory(conn)
10808
+ conn.commit()
10809
+
10810
+
10271
10811
  # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
10272
10812
 
10273
10813
  @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
@@ -11342,15 +11882,30 @@ def cmd_db_status(args: argparse.Namespace) -> int:
11342
11882
  "migrations": [],
11343
11883
  "_open_error": str(exc),
11344
11884
  }
11885
+ # #1000 §5.10: the conversations leg opens under the diagnostic read
11886
+ # admission and DECLINES while the probe, recovery or another maintenance
11887
+ # holder owns the family; report that the same way.
11888
+ try:
11889
+ conversations_status = _db_status_for(
11890
+ _cctally_core.CONVERSATIONS_DB_PATH, _CONVERSATIONS_MIGRATIONS,
11891
+ "conversations.db",
11892
+ )
11893
+ # An admission whose lock file cannot be opened (an unwritable data
11894
+ # folder) is a decline as well, never a traceback.
11895
+ except (sqlite3.DatabaseError, OSError) as exc:
11896
+ conversations_status = {
11897
+ "path": str(_cctally_core.CONVERSATIONS_DB_PATH),
11898
+ "user_version": 0,
11899
+ "registry_size": len(_CONVERSATIONS_MIGRATIONS),
11900
+ "migrations": [],
11901
+ "_open_error": str(exc),
11902
+ }
11345
11903
  payload = {
11346
11904
  "schema_version": 1,
11347
11905
  "databases": {
11348
11906
  "stats.db": stats_status,
11349
11907
  "cache.db": _db_status_for(_cctally_core.CACHE_DB_PATH, _CACHE_MIGRATIONS, "cache.db"),
11350
- "conversations.db": _db_status_for(
11351
- _cctally_core.CONVERSATIONS_DB_PATH, _CONVERSATIONS_MIGRATIONS,
11352
- "conversations.db",
11353
- ),
11908
+ "conversations.db": conversations_status,
11354
11909
  },
11355
11910
  }
11356
11911
  # #997 D10: additive — each detached rebuild worker's state and its
@@ -11428,15 +11983,7 @@ def _db_status_for(
11428
11983
  except sqlite3.OperationalError.
11429
11984
  """
11430
11985
  if not db_path.exists():
11431
- return {
11432
- "path": str(db_path),
11433
- "user_version": 0,
11434
- "registry_size": len(registry),
11435
- "migrations": [
11436
- {"seq": m.seq, "name": m.name, "status": "pending"}
11437
- for m in registry
11438
- ],
11439
- }
11986
+ return _db_status_absent(db_path, registry)
11440
11987
  if db_path == _cctally_core.DB_PATH:
11441
11988
  # #386: participate in the replacement protocol, and connect `mode=rw`
11442
11989
  # so this diagnostic can never MATERIALISE the very stats.db a
@@ -11452,34 +11999,77 @@ def _db_status_for(
11452
11999
  f"file:{p}?mode=rw", uri=True, **kw),
11453
12000
  recover_interruptions=recover_interrupted_stats,
11454
12001
  )
12002
+ elif db_label == "conversations.db":
12003
+ # #1000 §5.10: the diagnostic read admission (maintenance SHARED for
12004
+ # the connection's life, `mode=ro`), so this diagnostic can neither
12005
+ # open the family under the pre-rebuild probe's in-place read nor
12006
+ # recreate a main file a quarantine just renamed: the `exists()`
12007
+ # check above is a TOCTOU, and a plain `sqlite3.connect` recreates a
12008
+ # file removed in the gap. A decline raises `sqlite3.DatabaseError`
12009
+ # with the readers' existing text, which `cmd_db_status` renders as
12010
+ # this leg's `_open_error`.
12011
+ import _cctally_cache
12012
+
12013
+ with _cctally_cache.conversation_ro_guarded(timeout=5.0) as conn:
12014
+ if conn is None:
12015
+ if not os.path.lexists(db_path):
12016
+ return _db_status_absent(db_path, registry)
12017
+ raise sqlite3.DatabaseError(
12018
+ _cctally_cache.conversation_ro_decline_reason())
12019
+ return _db_status_rows(conn, db_path, registry, db_label)
11455
12020
  else:
11456
12021
  conn = sqlite3.connect(db_path)
11457
12022
  try:
11458
- user_version = conn.execute("PRAGMA user_version").fetchone()[0]
11459
- # Tolerate missing tables (e.g., cache.db never opened by framework).
11460
- # Alias legacy unprefixed names to their NNN_ equivalents so a
11461
- # `db status` that runs before any open_db()-via-bootstrap rename
11462
- # still reports legacy-marker DBs as applied, not pending.
11463
- aliases = _LEGACY_MARKER_ALIASES_BY_DB.get(db_label, {})
11464
- applied_rows: dict[str, str] = {}
11465
- try:
11466
- for row in conn.execute(
11467
- "SELECT name, applied_at_utc FROM schema_migrations"
11468
- ).fetchall():
11469
- applied_rows[aliases.get(row[0], row[0])] = row[1]
11470
- except sqlite3.OperationalError:
11471
- pass
11472
- skipped_rows: dict[str, tuple[str, str | None]] = {}
11473
- try:
11474
- for row in conn.execute(
11475
- "SELECT name, skipped_at_utc, reason FROM schema_migrations_skipped"
11476
- ).fetchall():
11477
- skipped_rows[row[0]] = (row[1], row[2])
11478
- except sqlite3.OperationalError:
11479
- pass
12023
+ return _db_status_rows(conn, db_path, registry, db_label)
11480
12024
  finally:
11481
12025
  conn.close()
11482
12026
 
12027
+
12028
+ def _db_status_absent(
12029
+ db_path: pathlib.Path, registry: list[Migration],
12030
+ ) -> dict:
12031
+ """The status dict for an absent store: every migration pending."""
12032
+ return {
12033
+ "path": str(db_path),
12034
+ "user_version": 0,
12035
+ "registry_size": len(registry),
12036
+ "migrations": [
12037
+ {"seq": m.seq, "name": m.name, "status": "pending"}
12038
+ for m in registry
12039
+ ],
12040
+ }
12041
+
12042
+
12043
+ def _db_status_rows(
12044
+ conn: sqlite3.Connection,
12045
+ db_path: pathlib.Path,
12046
+ registry: list[Migration],
12047
+ db_label: str,
12048
+ ) -> dict:
12049
+ """The status dict for ``db_path`` read through the open ``conn``."""
12050
+ user_version = conn.execute("PRAGMA user_version").fetchone()[0]
12051
+ # Tolerate missing tables (e.g., cache.db never opened by framework).
12052
+ # Alias legacy unprefixed names to their NNN_ equivalents so a
12053
+ # `db status` that runs before any open_db()-via-bootstrap rename
12054
+ # still reports legacy-marker DBs as applied, not pending.
12055
+ aliases = _LEGACY_MARKER_ALIASES_BY_DB.get(db_label, {})
12056
+ applied_rows: dict[str, str] = {}
12057
+ try:
12058
+ for row in conn.execute(
12059
+ "SELECT name, applied_at_utc FROM schema_migrations"
12060
+ ).fetchall():
12061
+ applied_rows[aliases.get(row[0], row[0])] = row[1]
12062
+ except sqlite3.OperationalError:
12063
+ pass
12064
+ skipped_rows: dict[str, tuple[str, str | None]] = {}
12065
+ try:
12066
+ for row in conn.execute(
12067
+ "SELECT name, skipped_at_utc, reason FROM schema_migrations_skipped"
12068
+ ).fetchall():
12069
+ skipped_rows[row[0]] = (row[1], row[2])
12070
+ except sqlite3.OperationalError:
12071
+ pass
12072
+
11483
12073
  failed_names = _db_status_failed_names_from_log(db_label)
11484
12074
  migrations_out: list[dict] = []
11485
12075
  for m in registry:
@@ -11631,6 +12221,12 @@ def _acquire_db_admin_writer_flocks(
11631
12221
  ``PRAGMA user_version = 0`` against the live stats index. stats.db now takes
11632
12222
  its maintenance lock EXCLUSIVE, which is the regime spec section 3.1 puts
11633
12223
  administrative mutation in.
12224
+
12225
+ #1000 §5.8: conversations.db takes, in the conversation-writer lock order
12226
+ (maintenance before the providers, Claude then Codex), its maintenance
12227
+ lock SHARED and both provider locks EXCLUSIVE, as every conversation
12228
+ writer does, so `db skip` / `db unskip` can neither write the store under
12229
+ the pre-rebuild probe's in-place read nor across recovery's quarantine.
11634
12230
  """
11635
12231
  from _lib_cache_writer_lock import acquire_ordered_flocks
11636
12232
 
@@ -11650,6 +12246,16 @@ def _acquire_db_admin_writer_flocks(
11650
12246
  if held is not None:
11651
12247
  _cctally_core.note_stats_maintenance_acquired()
11652
12248
  return held
12249
+ if db_label == "conversations.db":
12250
+ return acquire_ordered_flocks(
12251
+ [
12252
+ (_cctally_core.CONVERSATIONS_LOCK_MAINTENANCE_PATH,
12253
+ fcntl.LOCK_SH),
12254
+ (_cctally_core.CONVERSATIONS_LOCK_PATH, fcntl.LOCK_EX),
12255
+ (_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, fcntl.LOCK_EX),
12256
+ ],
12257
+ timeout=timeout,
12258
+ )
11653
12259
  return []
11654
12260
 
11655
12261
 
@@ -11976,11 +12582,49 @@ def _encode_repair_owner(
11976
12582
  ) + "\n"
11977
12583
 
11978
12584
 
11979
- def _read_repair_owner(marker: pathlib.Path) -> "dict[str, Any] | None":
12585
+ #: #1020: the record file inside a directory-form owner record, and how many
12586
+ #: times a reader resolves and reads it before reporting it unreadable.
12587
+ _OWNER_RECORD_FILE_NAME = "owner"
12588
+ _OWNER_RECORD_READ_ATTEMPTS = 3
12589
+
12590
+
12591
+ def _owner_record_file(marker: pathlib.Path) -> pathlib.Path:
12592
+ """Return the file that holds ``marker``'s owner record (#1020).
12593
+
12594
+ A marker published where hard links work is the record itself; one
12595
+ published on a volume without them is a directory holding ``owner``. A
12596
+ failed ``lstat`` resolves to the marker, so the read fails as before."""
11980
12597
  try:
11981
- raw = marker.read_text().strip()
12598
+ st = os.lstat(marker)
11982
12599
  except OSError:
11983
- return None
12600
+ return marker
12601
+ if stat.S_ISDIR(st.st_mode):
12602
+ return marker / _OWNER_RECORD_FILE_NAME
12603
+ return marker
12604
+
12605
+
12606
+ def _read_owner_record_text(
12607
+ marker: pathlib.Path,
12608
+ ) -> "tuple[str, pathlib.Path] | None":
12609
+ """Read ``marker``'s owner record; return its stripped text and the file
12610
+ read, or ``None`` when it cannot be read.
12611
+
12612
+ A read that finds the other form (``IsADirectoryError`` /
12613
+ ``NotADirectoryError``) means one owner released the record and another
12614
+ published it in the other form between the ``lstat`` and the read, so the
12615
+ record is resolved and read again, a bounded number of times."""
12616
+ for _attempt in range(_OWNER_RECORD_READ_ATTEMPTS):
12617
+ record = _owner_record_file(marker)
12618
+ try:
12619
+ return record.read_text().strip(), record
12620
+ except (IsADirectoryError, NotADirectoryError):
12621
+ continue
12622
+ except OSError:
12623
+ return None
12624
+ return None
12625
+
12626
+
12627
+ def _decode_repair_owner(raw: str) -> "dict[str, Any] | None":
11984
12628
  def legacy_owner() -> "dict[str, Any] | None":
11985
12629
  # Legacy markers contained only a PID. They remain live for a bounded
11986
12630
  # lease when that PID exists, preventing a reused PID from wedging the
@@ -12012,8 +12656,22 @@ def _read_repair_owner(marker: pathlib.Path) -> "dict[str, Any] | None":
12012
12656
  return decoded
12013
12657
 
12014
12658
 
12659
+ def _read_repair_owner_record(
12660
+ marker: pathlib.Path,
12661
+ ) -> "tuple[dict[str, Any] | None, pathlib.Path | None]":
12662
+ read = _read_owner_record_text(marker)
12663
+ if read is None:
12664
+ return None, None
12665
+ raw, record = read
12666
+ return _decode_repair_owner(raw), record
12667
+
12668
+
12669
+ def _read_repair_owner(marker: pathlib.Path) -> "dict[str, Any] | None":
12670
+ return _read_repair_owner_record(marker)[0]
12671
+
12672
+
12015
12673
  def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
12016
- owner = _read_repair_owner(marker)
12674
+ owner, record = _read_repair_owner_record(marker)
12017
12675
  if owner is None:
12018
12676
  return False, "malformed owner record"
12019
12677
  pid = int(owner["pid"])
@@ -12026,8 +12684,11 @@ def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
12026
12684
  if actual_start == expected_start:
12027
12685
  return True, f"pid {pid}"
12028
12686
  return False, f"pid {pid} was reused"
12687
+ # #1020: the lease is the age of the record file that was read, never a
12688
+ # directory marker's own mtime (an AppleDouble companion moves it); a
12689
+ # record file gone since the read was released and is not live.
12029
12690
  try:
12030
- age_seconds = max(0.0, time.time() - marker.stat().st_mtime)
12691
+ age_seconds = max(0.0, time.time() - record.stat().st_mtime)
12031
12692
  except OSError:
12032
12693
  return False, "owner record disappeared"
12033
12694
  if age_seconds <= _REPAIR_OWNER_UNKNOWN_IDENTITY_LEASE_SECONDS:
@@ -12035,21 +12696,94 @@ def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
12035
12696
  return False, f"pid {pid} owner identity lease expired"
12036
12697
 
12037
12698
 
12699
+ def _remove_owner_record(marker: pathlib.Path) -> None:
12700
+ """Remove ``marker``'s owner record in either form (#1020).
12701
+
12702
+ A file (by ``lstat``, so a symlink is never followed) is unlinked. A
12703
+ directory is renamed to a unique private tomb, the one step readers
12704
+ observe; the parent is fsynced; only then is the tomb deleted, best effort.
12705
+ An already-absent marker is not an error. An error at the parent fsync
12706
+ propagates after the name is gone and leaves the tomb, so cleanup never
12707
+ precedes the barrier."""
12708
+ try:
12709
+ st = os.lstat(marker)
12710
+ except (FileNotFoundError, NotADirectoryError):
12711
+ return
12712
+ tomb = None
12713
+ try:
12714
+ if stat.S_ISDIR(st.st_mode):
12715
+ tomb = marker.with_name(
12716
+ f".{marker.name}.{os.getpid()}-{time.time_ns()}-"
12717
+ f"{os.urandom(4).hex()}.removed"
12718
+ )
12719
+ os.rename(marker, tomb)
12720
+ else:
12721
+ os.unlink(marker)
12722
+ except FileNotFoundError:
12723
+ return
12724
+ _fsync_directory(marker.parent)
12725
+ if tomb is not None:
12726
+ shutil.rmtree(tomb, ignore_errors=True)
12727
+
12728
+
12729
+ def _open_owner_record_lock_fd(marker: pathlib.Path) -> int:
12730
+ """Open (creating) ``<marker>.lock``, the removal lock's file (#1020),
12731
+ with the maintenance-lock idiom of
12732
+ ``_cctally_cache._open_maintenance_lock_fd``; the caller owns the fd."""
12733
+ lock_path = marker.with_name(f"{marker.name}.lock")
12734
+ flags = os.O_RDWR | os.O_CREAT | os.O_APPEND | getattr(os, "O_CLOEXEC", 0)
12735
+ try:
12736
+ return os.open(lock_path, flags, 0o666)
12737
+ except FileNotFoundError:
12738
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
12739
+ return os.open(lock_path, flags, 0o666)
12740
+
12741
+
12742
+ @contextlib.contextmanager
12743
+ def _owner_record_removal_lock(marker: pathlib.Path) -> "Iterator[None]":
12744
+ """Hold ``<marker>.lock`` exclusive for one removal of ``marker`` (#1020).
12745
+
12746
+ Every remover (both stale reclaimers, both releases) holds it across its
12747
+ absence check, read, judgment and removal, so the record it judged is the
12748
+ one it removes: while a record is present only a removal (which waits
12749
+ here) or nothing can change it, because publication needs the name
12750
+ absent. Publication never takes it. It is a leaf: nothing is acquired
12751
+ under it, and no caller holding it calls anything that takes it again
12752
+ (``flock`` conflicts between two descriptors of one process). Opened with
12753
+ the maintenance-lock idiom of ``_cctally_cache._open_maintenance_lock_fd``,
12754
+ on its own descriptor per acquisition."""
12755
+ fd = _open_owner_record_lock_fd(marker)
12756
+ try:
12757
+ fcntl.flock(fd, fcntl.LOCK_EX)
12758
+ yield
12759
+ finally:
12760
+ os.close(fd)
12761
+
12762
+
12763
+ def _owner_record_absent(marker: pathlib.Path) -> bool:
12764
+ """Whether ``marker``'s name is absent, by ``lstat`` (a dangling symlink
12765
+ is present). Only ``FileNotFoundError`` / ``NotADirectoryError`` prove
12766
+ absence; every other lookup error propagates, never read as absence."""
12767
+ try:
12768
+ os.lstat(marker)
12769
+ except (FileNotFoundError, NotADirectoryError):
12770
+ return True
12771
+ return False
12772
+
12773
+
12038
12774
  def _remove_stale_repair_marker(path: pathlib.Path) -> "tuple[bool, str]":
12039
12775
  """Remove a provably stale marker; the caller owns maintenance-exclusive."""
12040
12776
  marker = _repair_marker_path(path)
12041
- if not marker.exists():
12042
- return True, ""
12043
- live, reason = _repair_marker_is_live(marker)
12044
- if live:
12045
- return False, f"another {path.name} repair owns {marker} ({reason})"
12046
12777
  try:
12047
- marker.unlink()
12048
- except FileNotFoundError:
12049
- return True, ""
12778
+ with _owner_record_removal_lock(marker):
12779
+ if _owner_record_absent(marker):
12780
+ return True, ""
12781
+ live, reason = _repair_marker_is_live(marker)
12782
+ if live:
12783
+ return False, f"another {path.name} repair owns {marker} ({reason})"
12784
+ _remove_owner_record(marker)
12050
12785
  except OSError as exc:
12051
12786
  return False, f"could not remove stale repair marker {marker}: {exc}"
12052
- _fsync_directory(marker.parent)
12053
12787
  return True, reason
12054
12788
 
12055
12789
 
@@ -12065,6 +12799,96 @@ def _claim_repair_marker(
12065
12799
  )
12066
12800
 
12067
12801
 
12802
+ def _publish_owner_record_link(temp: pathlib.Path, marker: pathlib.Path) -> None:
12803
+ """Hard-link the private record ``temp`` onto ``marker`` (#1020).
12804
+
12805
+ While ``PYTEST_CURRENT_TEST`` is set, ``CCTALLY_TEST_OWNER_RECORD_LINK_ERRNO``
12806
+ names an errno the link raises instead, so a child-process test can stand
12807
+ in for a volume without hard links through the same classification a real
12808
+ ``link(2)`` refusal takes. Production never reads it."""
12809
+ if os.environ.get("PYTEST_CURRENT_TEST"):
12810
+ name = os.environ.get("CCTALLY_TEST_OWNER_RECORD_LINK_ERRNO")
12811
+ if name:
12812
+ code = getattr(errno, name)
12813
+ raise OSError(code, os.strerror(code), str(temp), None, str(marker))
12814
+ os.link(temp, marker)
12815
+
12816
+
12817
+ def _trace_owner_record_publication(marker: pathlib.Path, form: str) -> None:
12818
+ """Test-only (#1020): record which form (``file`` or ``directory``)
12819
+ published ``marker``, as ``<marker name> <form>`` lines appended to the
12820
+ file ``CCTALLY_TEST_OWNER_RECORD_TRACE`` names, only while
12821
+ ``PYTEST_CURRENT_TEST`` is set."""
12822
+ if not os.environ.get("PYTEST_CURRENT_TEST"):
12823
+ return
12824
+ trace = os.environ.get("CCTALLY_TEST_OWNER_RECORD_TRACE")
12825
+ if not trace:
12826
+ return
12827
+ with open(trace, "a", encoding="utf-8") as fh:
12828
+ fh.write(f"{marker.name} {form}\n")
12829
+
12830
+
12831
+ #: #1020: a directory-form publication whose rename finds the marker name
12832
+ #: taken — a populated directory (``EEXIST`` / ``ENOTEMPTY``) or a file
12833
+ #: (``ENOTDIR``) — is contention, handled as ``FileExistsError`` on a link.
12834
+ _OWNER_RECORD_RENAME_CONTENTION = frozenset({
12835
+ errno.EEXIST, errno.ENOTEMPTY, errno.ENOTDIR,
12836
+ })
12837
+
12838
+
12839
+ def _owner_record_link_unsupported(exc: OSError) -> bool:
12840
+ """Whether the publishing link failed because the volume has no hard
12841
+ links (#1020): ``ENOTSUP`` / ``EOPNOTSUPP`` anywhere, and ``EPERM`` on
12842
+ Linux, where ``link(2)`` documents it for filesystems without hard links
12843
+ (vfat, exfat). Every other errno, macOS ``EPERM`` included, is a real
12844
+ failure."""
12845
+ if exc.errno in (errno.ENOTSUP, errno.EOPNOTSUPP):
12846
+ return True
12847
+ return exc.errno == errno.EPERM and sys.platform.startswith("linux")
12848
+
12849
+
12850
+ def _publish_owner_record_directory(
12851
+ marker: pathlib.Path, payload: bytes, claim_id: str,
12852
+ ) -> bool:
12853
+ """Publish ``payload`` at ``marker`` as a directory holding ``owner``
12854
+ (#1020), for a volume without hard links; ``False`` when the name is taken.
12855
+
12856
+ The record is written and fsynced inside a private directory, which is
12857
+ fsynced and renamed onto the marker: rename(2) never replaces a populated
12858
+ directory, so the record is complete or absent and exactly one claimant
12859
+ wins, as with the hard link. The private directory never outlives the
12860
+ attempt."""
12861
+ private = marker.with_name(f".{marker.name}.{claim_id}.d")
12862
+ os.mkdir(private, 0o700)
12863
+ try:
12864
+ fd = os.open(
12865
+ private / _OWNER_RECORD_FILE_NAME,
12866
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL,
12867
+ 0o600,
12868
+ )
12869
+ try:
12870
+ os.write(fd, payload)
12871
+ os.fsync(fd)
12872
+ finally:
12873
+ os.close(fd)
12874
+ _fsync_directory(private)
12875
+ try:
12876
+ os.rename(private, marker)
12877
+ except OSError as exc:
12878
+ if exc.errno in _OWNER_RECORD_RENAME_CONTENTION:
12879
+ return False
12880
+ raise
12881
+ _fsync_directory(marker.parent)
12882
+ return True
12883
+ finally:
12884
+ try:
12885
+ os.lstat(private)
12886
+ except OSError:
12887
+ pass
12888
+ else:
12889
+ shutil.rmtree(private, ignore_errors=True)
12890
+
12891
+
12068
12892
  def _claim_owner_record(
12069
12893
  marker: pathlib.Path,
12070
12894
  remove_stale: "Callable[[], tuple[bool, str]]",
@@ -12077,10 +12901,18 @@ def _claim_owner_record(
12077
12901
  The repair marker's discipline, shared with #901's keeper-yield request:
12078
12902
  the record is written and fsynced under a private name and linked into
12079
12903
  place, so it is complete or absent; an existing record is replaced only
12080
- when ``remove_stale`` proves its owner dead or its pid reused."""
12904
+ when ``remove_stale`` proves its owner dead or its pid reused. Where the
12905
+ volume has no hard links (#1020) the same record is published as a
12906
+ directory holding it (`_publish_owner_record_directory`); the hard link
12907
+ is tried first on every claim."""
12081
12908
  process_start = _process_start_identity(os.getpid())
12082
12909
  if process_start is None:
12083
12910
  raise OSError(f"could not establish {owner} process identity")
12911
+ # #1020: the removal lock's file exists before the record does, so this
12912
+ # claim's release never needs a new directory entry (a full disk would
12913
+ # otherwise leave this process's own record behind). Creating it is not
12914
+ # taking the lock; publication never takes it.
12915
+ os.close(_open_owner_record_lock_fd(marker))
12084
12916
  for _attempt in range(3):
12085
12917
  claim_id = f"{os.getpid()}-{time.time_ns()}-{os.urandom(8).hex()}"
12086
12918
  temp = marker.with_name(f".{marker.name}.{claim_id}.tmp")
@@ -12096,14 +12928,32 @@ def _claim_owner_record(
12096
12928
  finally:
12097
12929
  os.close(fd)
12098
12930
  try:
12931
+ link_unsupported = False
12099
12932
  try:
12100
- os.link(temp, marker)
12933
+ _publish_owner_record_link(temp, marker)
12101
12934
  except FileExistsError:
12102
12935
  removed, reason = remove_stale()
12103
12936
  if not removed:
12104
12937
  return None, reason
12105
12938
  continue
12106
- _fsync_directory(marker.parent)
12939
+ except OSError as exc:
12940
+ if not _owner_record_link_unsupported(exc):
12941
+ raise
12942
+ link_unsupported = True
12943
+ if link_unsupported:
12944
+ # #1020: no hard links here (exFAT); publish the same
12945
+ # complete record as a directory, by an exclusive rename.
12946
+ if not _publish_owner_record_directory(
12947
+ marker, payload.encode("utf-8"), claim_id,
12948
+ ):
12949
+ removed, reason = remove_stale()
12950
+ if not removed:
12951
+ return None, reason
12952
+ continue
12953
+ _trace_owner_record_publication(marker, "directory")
12954
+ else:
12955
+ _fsync_directory(marker.parent)
12956
+ _trace_owner_record_publication(marker, "file")
12107
12957
  return RepairMarkerClaim(marker=marker, claim_id=claim_id), ""
12108
12958
  finally:
12109
12959
  try:
@@ -12115,14 +12965,15 @@ def _claim_owner_record(
12115
12965
 
12116
12966
  def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None:
12117
12967
  marker = _repair_marker_path(path)
12118
- owner = _read_repair_owner(marker)
12119
- if owner is not None and owner.get("claim_id") != claim.claim_id:
12120
- raise OSError(f"repair marker ownership changed before cleanup: {marker}")
12121
- try:
12122
- marker.unlink()
12123
- except FileNotFoundError:
12124
- return
12125
- _fsync_directory(marker.parent)
12968
+ with _owner_record_removal_lock(marker):
12969
+ # #1020: an absent name was already reclaimed; a newer owner may
12970
+ # publish there at any moment, so it is neither read nor removed.
12971
+ if _owner_record_absent(marker):
12972
+ return
12973
+ owner = _read_repair_owner(marker)
12974
+ if owner is not None and owner.get("claim_id") != claim.claim_id:
12975
+ raise OSError(f"repair marker ownership changed before cleanup: {marker}")
12976
+ _remove_owner_record(marker)
12126
12977
 
12127
12978
 
12128
12979
  #: #901 W9 (Q15): the one monotonic deadline a process that needs a
@@ -12141,18 +12992,16 @@ def _remove_stale_keeper_yield_request(
12141
12992
  """Remove a provably stale keeper-yield request (dead or reused-pid
12142
12993
  owner), as `_remove_stale_repair_marker` does for the repair marker."""
12143
12994
  request = _keeper_yield_request_path(path)
12144
- if not request.exists():
12145
- return True, ""
12146
- live, reason = _repair_marker_is_live(request)
12147
- if live:
12148
- return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
12149
12995
  try:
12150
- request.unlink()
12151
- except FileNotFoundError:
12152
- return True, ""
12996
+ with _owner_record_removal_lock(request):
12997
+ if _owner_record_absent(request):
12998
+ return True, ""
12999
+ live, reason = _repair_marker_is_live(request)
13000
+ if live:
13001
+ return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
13002
+ _remove_owner_record(request)
12153
13003
  except OSError as exc:
12154
13004
  return False, f"could not remove stale keeper-yield request {request}: {exc}"
12155
- _fsync_directory(request.parent)
12156
13005
  return True, reason
12157
13006
 
12158
13007
 
@@ -12175,17 +13024,16 @@ def _release_keeper_yield_request(
12175
13024
  owns is left alone. Best effort: a request left behind fences keepers
12176
13025
  only while this process lives, and it is stale once the process exits."""
12177
13026
  request = _keeper_yield_request_path(path)
12178
- owner = _read_repair_owner(request)
12179
- if owner is not None and owner.get("claim_id") != claim.claim_id:
12180
- return
12181
13027
  try:
12182
- request.unlink()
13028
+ with _owner_record_removal_lock(request):
13029
+ if _owner_record_absent(request):
13030
+ return
13031
+ owner = _read_repair_owner(request)
13032
+ if owner is not None and owner.get("claim_id") != claim.claim_id:
13033
+ return
13034
+ _remove_owner_record(request)
12183
13035
  except OSError:
12184
13036
  return
12185
- try:
12186
- _fsync_directory(request.parent)
12187
- except OSError:
12188
- pass
12189
13037
 
12190
13038
 
12191
13039
  @contextlib.contextmanager