cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
package/bin/_cctally_db.py
CHANGED
|
@@ -64,6 +64,7 @@ import contextlib
|
|
|
64
64
|
import contextvars
|
|
65
65
|
import datetime as dt
|
|
66
66
|
import enum
|
|
67
|
+
import errno
|
|
67
68
|
import fcntl
|
|
68
69
|
import json
|
|
69
70
|
import os
|
|
@@ -71,6 +72,7 @@ import pathlib
|
|
|
71
72
|
import re
|
|
72
73
|
import shutil
|
|
73
74
|
import sqlite3
|
|
75
|
+
import stat
|
|
74
76
|
import subprocess
|
|
75
77
|
import sys
|
|
76
78
|
import tempfile
|
|
@@ -1068,6 +1070,8 @@ def write_corruption_forensics(
|
|
|
1068
1070
|
trigger_origin: "str | None" = None,
|
|
1069
1071
|
trigger_exception: "BaseException | None" = None,
|
|
1070
1072
|
return_result: bool = False,
|
|
1073
|
+
probe_immutable: bool = False,
|
|
1074
|
+
before_publish: "Callable[[], None] | None" = None,
|
|
1071
1075
|
) -> "pathlib.Path | None | CorruptionForensicsResult":
|
|
1072
1076
|
"""Write the #336-defense-2 corruption forensics bundle FIRST — before any
|
|
1073
1077
|
quarantine/rebuild disturbs the evidence (spec §6.3). Captures the file family
|
|
@@ -1083,6 +1087,15 @@ def write_corruption_forensics(
|
|
|
1083
1087
|
isolated family snapshot while the bundle still identifies the live
|
|
1084
1088
|
``db_path``. Probe/write unavailability is always unconfirmed. Confirmed
|
|
1085
1089
|
bundles are atomically replaced and fsynced before this function returns.
|
|
1090
|
+
|
|
1091
|
+
#1000: ``probe_immutable=True`` opens the integrity connection on
|
|
1092
|
+
``probe_db_path`` as ``mode=ro&immutable=1`` through an escaped absolute
|
|
1093
|
+
URI, for a caller that checks a live main file in place: a plain
|
|
1094
|
+
``mode=ro`` open of a WAL-mode store creates ``-wal``/``-shm`` beside it.
|
|
1095
|
+
``before_publish``, when given, runs after the integrity probe and the
|
|
1096
|
+
damage scan and immediately before the bundle is published; an exception
|
|
1097
|
+
it raises propagates (it is outside every handler here) and nothing is
|
|
1098
|
+
published for that attempt.
|
|
1086
1099
|
"""
|
|
1087
1100
|
if (trigger_origin is None) != (trigger_exception is None):
|
|
1088
1101
|
raise ValueError(
|
|
@@ -1157,8 +1170,15 @@ def write_corruption_forensics(
|
|
|
1157
1170
|
}
|
|
1158
1171
|
except OSError:
|
|
1159
1172
|
bundle["family"][p.name] = None
|
|
1173
|
+
if probe_immutable:
|
|
1174
|
+
probe_uri = (
|
|
1175
|
+
pathlib.Path(os.path.abspath(probe_db_path)).as_uri()
|
|
1176
|
+
+ "?mode=ro&immutable=1"
|
|
1177
|
+
)
|
|
1178
|
+
else:
|
|
1179
|
+
probe_uri = f"file:{probe_db_path}?mode=ro"
|
|
1160
1180
|
try:
|
|
1161
|
-
c = sqlite3.connect(
|
|
1181
|
+
c = sqlite3.connect(probe_uri, uri=True)
|
|
1162
1182
|
try:
|
|
1163
1183
|
c.execute("PRAGMA busy_timeout=2000")
|
|
1164
1184
|
bundle["integrityCheck"] = [
|
|
@@ -1214,6 +1234,8 @@ def write_corruption_forensics(
|
|
|
1214
1234
|
bundle["holders"].append(line.strip())
|
|
1215
1235
|
except Exception as exc:
|
|
1216
1236
|
bundle["holders"] = [f"error: {exc}"]
|
|
1237
|
+
if before_publish is not None:
|
|
1238
|
+
before_publish()
|
|
1217
1239
|
try:
|
|
1218
1240
|
_atomic_write_private_json(out, bundle)
|
|
1219
1241
|
except OSError:
|
|
@@ -4380,6 +4402,210 @@ def _apply_codex_accounting_change_ledger(conn: sqlite3.Connection) -> None:
|
|
|
4380
4402
|
conn.execute(statement)
|
|
4381
4403
|
|
|
4382
4404
|
|
|
4405
|
+
# ── #927 (#981 Part B): the retained-identity inventory (spec §4.2 B1) ───────
|
|
4406
|
+
#
|
|
4407
|
+
# The pre-walk prune inventory used to read every retained Codex source
|
|
4408
|
+
# identity from four history-sized families on every hook (13.2-13.5 s cold on
|
|
4409
|
+
# the measured store). This table holds each distinct ``(source_path,
|
|
4410
|
+
# source_root_key)`` those families have carried, maintained by INSERT and
|
|
4411
|
+
# identity-UPDATE triggers in the same transaction as the write. There are no
|
|
4412
|
+
# DELETE triggers: deletion owners reconcile the table themselves, and every
|
|
4413
|
+
# other deletion leaves a SUPERSET, which the planner confirms with indexed
|
|
4414
|
+
# existence probes before an identity becomes protection evidence.
|
|
4415
|
+
#
|
|
4416
|
+
# A NULL root is distinct from every non-NULL root, and ``''`` from NULL, so
|
|
4417
|
+
# the uniqueness is an expression index rather than a plain UNIQUE. Each
|
|
4418
|
+
# trigger body inserts only when the identity is absent: an outer
|
|
4419
|
+
# statement's conflict policy propagates into trigger-body INSERTs (the #582
|
|
4420
|
+
# ledger note above), so the ``NOT EXISTS`` keeps a conflict from ever being
|
|
4421
|
+
# raised whatever that policy is.
|
|
4422
|
+
#
|
|
4423
|
+
# ``cache_meta.codex_identity_inventory`` is ``ready`` or
|
|
4424
|
+
# ``bootstrap:<family>:<last_rowid>``; any other value reads as
|
|
4425
|
+
# ``bootstrap:files:0``. Only ``ready`` lets the planner read the table.
|
|
4426
|
+
|
|
4427
|
+
CODEX_IDENTITY_INVENTORY_KEY = "codex_identity_inventory"
|
|
4428
|
+
CODEX_IDENTITY_INVENTORY_READY = "ready"
|
|
4429
|
+
#: The bootstrap's fixed family order, each with its rowid source query.
|
|
4430
|
+
CODEX_IDENTITY_INVENTORY_FAMILIES: "tuple[tuple[str, str, str, str], ...]" = (
|
|
4431
|
+
# (state token, table, path column, extra WHERE)
|
|
4432
|
+
("files", "codex_session_files", "path", ""),
|
|
4433
|
+
("entries", "codex_session_entries", "source_path", ""),
|
|
4434
|
+
("quota", "quota_window_snapshots", "source_path", " AND source = 'codex'"),
|
|
4435
|
+
("threads", "codex_conversation_threads", "source_path", ""),
|
|
4436
|
+
)
|
|
4437
|
+
|
|
4438
|
+
_CODEX_RETAINED_IDENTITY_OBJECTS = (
|
|
4439
|
+
("table", "codex_retained_identities"),
|
|
4440
|
+
("index", "idx_codex_retained_identities_key"),
|
|
4441
|
+
("trigger", "trg_codex_identity_files_ins"),
|
|
4442
|
+
("trigger", "trg_codex_identity_files_upd"),
|
|
4443
|
+
("trigger", "trg_codex_identity_entries_ins"),
|
|
4444
|
+
("trigger", "trg_codex_identity_entries_upd"),
|
|
4445
|
+
("trigger", "trg_codex_identity_quota_ins"),
|
|
4446
|
+
("trigger", "trg_codex_identity_quota_upd"),
|
|
4447
|
+
("trigger", "trg_codex_identity_threads_ins"),
|
|
4448
|
+
("trigger", "trg_codex_identity_threads_upd"),
|
|
4449
|
+
)
|
|
4450
|
+
|
|
4451
|
+
|
|
4452
|
+
def _codex_retained_identity_ddl() -> "tuple[str, ...]":
|
|
4453
|
+
def add(path: str) -> str:
|
|
4454
|
+
return f"""
|
|
4455
|
+
INSERT OR IGNORE INTO codex_retained_identities
|
|
4456
|
+
(source_path, source_root_key)
|
|
4457
|
+
SELECT NEW.{path}, NEW.source_root_key
|
|
4458
|
+
WHERE NOT EXISTS (
|
|
4459
|
+
SELECT 1 FROM codex_retained_identities
|
|
4460
|
+
WHERE source_path = NEW.{path}
|
|
4461
|
+
AND IFNULL(source_root_key, char(0))
|
|
4462
|
+
= IFNULL(NEW.source_root_key, char(0))
|
|
4463
|
+
);
|
|
4464
|
+
"""
|
|
4465
|
+
files, entries = add("path"), add("source_path")
|
|
4466
|
+
return (
|
|
4467
|
+
"""
|
|
4468
|
+
CREATE TABLE IF NOT EXISTS codex_retained_identities (
|
|
4469
|
+
source_path TEXT NOT NULL,
|
|
4470
|
+
source_root_key TEXT
|
|
4471
|
+
)
|
|
4472
|
+
""",
|
|
4473
|
+
"CREATE UNIQUE INDEX IF NOT EXISTS idx_codex_retained_identities_key "
|
|
4474
|
+
"ON codex_retained_identities"
|
|
4475
|
+
"(source_path, IFNULL(source_root_key, char(0)))",
|
|
4476
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_files_ins",
|
|
4477
|
+
f"""
|
|
4478
|
+
CREATE TRIGGER trg_codex_identity_files_ins
|
|
4479
|
+
AFTER INSERT ON codex_session_files
|
|
4480
|
+
BEGIN {files} END
|
|
4481
|
+
""",
|
|
4482
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_files_upd",
|
|
4483
|
+
f"""
|
|
4484
|
+
CREATE TRIGGER trg_codex_identity_files_upd
|
|
4485
|
+
AFTER UPDATE OF path, source_root_key ON codex_session_files
|
|
4486
|
+
BEGIN {files} END
|
|
4487
|
+
""",
|
|
4488
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_entries_ins",
|
|
4489
|
+
f"""
|
|
4490
|
+
CREATE TRIGGER trg_codex_identity_entries_ins
|
|
4491
|
+
AFTER INSERT ON codex_session_entries
|
|
4492
|
+
BEGIN {entries} END
|
|
4493
|
+
""",
|
|
4494
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_entries_upd",
|
|
4495
|
+
f"""
|
|
4496
|
+
CREATE TRIGGER trg_codex_identity_entries_upd
|
|
4497
|
+
AFTER UPDATE OF source_path, source_root_key ON codex_session_entries
|
|
4498
|
+
BEGIN {entries} END
|
|
4499
|
+
""",
|
|
4500
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_quota_ins",
|
|
4501
|
+
f"""
|
|
4502
|
+
CREATE TRIGGER trg_codex_identity_quota_ins
|
|
4503
|
+
AFTER INSERT ON quota_window_snapshots
|
|
4504
|
+
WHEN NEW.source = 'codex'
|
|
4505
|
+
BEGIN {entries} END
|
|
4506
|
+
""",
|
|
4507
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_quota_upd",
|
|
4508
|
+
f"""
|
|
4509
|
+
CREATE TRIGGER trg_codex_identity_quota_upd
|
|
4510
|
+
AFTER UPDATE OF source, source_path, source_root_key
|
|
4511
|
+
ON quota_window_snapshots
|
|
4512
|
+
WHEN NEW.source = 'codex'
|
|
4513
|
+
BEGIN {entries} END
|
|
4514
|
+
""",
|
|
4515
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_threads_ins",
|
|
4516
|
+
f"""
|
|
4517
|
+
CREATE TRIGGER trg_codex_identity_threads_ins
|
|
4518
|
+
AFTER INSERT ON codex_conversation_threads
|
|
4519
|
+
BEGIN {entries} END
|
|
4520
|
+
""",
|
|
4521
|
+
"DROP TRIGGER IF EXISTS trg_codex_identity_threads_upd",
|
|
4522
|
+
f"""
|
|
4523
|
+
CREATE TRIGGER trg_codex_identity_threads_upd
|
|
4524
|
+
AFTER UPDATE OF source_path, source_root_key
|
|
4525
|
+
ON codex_conversation_threads
|
|
4526
|
+
BEGIN {entries} END
|
|
4527
|
+
""",
|
|
4528
|
+
)
|
|
4529
|
+
|
|
4530
|
+
|
|
4531
|
+
def _codex_identity_families_empty(conn: sqlite3.Connection) -> bool:
|
|
4532
|
+
"""True when none of the four families holds a Codex row (O(1) each)."""
|
|
4533
|
+
return not any(
|
|
4534
|
+
conn.execute(
|
|
4535
|
+
f"SELECT 1 FROM {table} WHERE 1{extra} LIMIT 1").fetchone()
|
|
4536
|
+
for _token, table, _path, extra in CODEX_IDENTITY_INVENTORY_FAMILIES
|
|
4537
|
+
)
|
|
4538
|
+
|
|
4539
|
+
|
|
4540
|
+
def _apply_codex_retained_identity_inventory(conn: sqlite3.Connection) -> None:
|
|
4541
|
+
"""Install the retained-identity inventory on every delivery path (B1).
|
|
4542
|
+
|
|
4543
|
+
Shared by ``_apply_cache_schema`` and migration 051 so fresh and upgraded
|
|
4544
|
+
stores converge. Atomic like the #901 latest summary: the table, its
|
|
4545
|
+
index, the eight triggers and the state key commit together, as a
|
|
4546
|
+
savepoint inside the caller's transaction or as an IMMEDIATE transaction
|
|
4547
|
+
of their own. Triggers come first, so no write can land between the
|
|
4548
|
+
state decision and the maintenance that keeps it true.
|
|
4549
|
+
|
|
4550
|
+
The state key is decided only when any of the ten objects was missing
|
|
4551
|
+
before this call (a store carrying all ten has been maintained since they
|
|
4552
|
+
were created, so its recorded state stands): ``ready`` when the four
|
|
4553
|
+
families hold no Codex row, else ``bootstrap:files:0`` for the off-hook
|
|
4554
|
+
backfill. A history scan never runs here. Guarded on the identity columns,
|
|
4555
|
+
which a legacy-shape cache whose schema apply returned early can lack.
|
|
4556
|
+
"""
|
|
4557
|
+
def _columns(table: str) -> "set[str]":
|
|
4558
|
+
return {
|
|
4559
|
+
str(row[0]) for row in conn.execute(
|
|
4560
|
+
"SELECT name FROM pragma_table_info(?)", (table,))
|
|
4561
|
+
}
|
|
4562
|
+
|
|
4563
|
+
if not (
|
|
4564
|
+
{"path", "source_root_key"} <= _columns("codex_session_files")
|
|
4565
|
+
and {"source_path", "source_root_key"} <= _columns(
|
|
4566
|
+
"codex_session_entries")
|
|
4567
|
+
and {"source", "source_path", "source_root_key"} <= _columns(
|
|
4568
|
+
"quota_window_snapshots")
|
|
4569
|
+
and {"source_path", "source_root_key"} <= _columns(
|
|
4570
|
+
"codex_conversation_threads")
|
|
4571
|
+
):
|
|
4572
|
+
return
|
|
4573
|
+
own_transaction = not conn.in_transaction
|
|
4574
|
+
conn.execute(
|
|
4575
|
+
"BEGIN IMMEDIATE" if own_transaction
|
|
4576
|
+
else "SAVEPOINT codex_retained_identity_apply")
|
|
4577
|
+
try:
|
|
4578
|
+
present = {
|
|
4579
|
+
(str(row[0]), str(row[1])) for row in conn.execute(
|
|
4580
|
+
"SELECT type, name FROM sqlite_master "
|
|
4581
|
+
"WHERE type IN ('table', 'index', 'trigger')")
|
|
4582
|
+
}
|
|
4583
|
+
complete = all(
|
|
4584
|
+
item in present for item in _CODEX_RETAINED_IDENTITY_OBJECTS)
|
|
4585
|
+
for statement in _codex_retained_identity_ddl():
|
|
4586
|
+
conn.execute(statement)
|
|
4587
|
+
if not complete:
|
|
4588
|
+
state = (
|
|
4589
|
+
CODEX_IDENTITY_INVENTORY_READY
|
|
4590
|
+
if _codex_identity_families_empty(conn)
|
|
4591
|
+
else "bootstrap:files:0")
|
|
4592
|
+
conn.execute(
|
|
4593
|
+
"INSERT INTO cache_meta(key, value) VALUES (?, ?) "
|
|
4594
|
+
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
|
4595
|
+
(CODEX_IDENTITY_INVENTORY_KEY, state))
|
|
4596
|
+
except BaseException:
|
|
4597
|
+
if own_transaction:
|
|
4598
|
+
conn.rollback()
|
|
4599
|
+
else:
|
|
4600
|
+
conn.execute("ROLLBACK TO codex_retained_identity_apply")
|
|
4601
|
+
conn.execute("RELEASE codex_retained_identity_apply")
|
|
4602
|
+
raise
|
|
4603
|
+
if own_transaction:
|
|
4604
|
+
conn.commit()
|
|
4605
|
+
else:
|
|
4606
|
+
conn.execute("RELEASE codex_retained_identity_apply")
|
|
4607
|
+
|
|
4608
|
+
|
|
4383
4609
|
# ── #901 W1: the maintained raw-partition maxima (spec §5.1, T1) ─────────────
|
|
4384
4610
|
#
|
|
4385
4611
|
# `load_codex_quota_observations(latest_per_identity=True)` computed
|
|
@@ -4808,6 +5034,46 @@ _CODEX_CONTAINER_CURSOR_COLUMNS: "tuple[tuple[str, str], ...]" = (
|
|
|
4808
5034
|
_CODEX_CONTAINER_CACHE_ONLY_COLUMNS: "tuple[tuple[str, str], ...]" = (
|
|
4809
5035
|
("container_resume_strikes", "INTEGER NOT NULL DEFAULT 0"),
|
|
4810
5036
|
)
|
|
5037
|
+
#: #944 spec 5.1/5.4: the derived tables of write-time Claude cost
|
|
5038
|
+
#: attribution. Both are journal-backed and disposable: the decisions replay
|
|
5039
|
+
#: from `claude_cost_account` ops, the timeline from admitted Claude
|
|
5040
|
+
#: `record-usage` observations.
|
|
5041
|
+
_CLAUDE_ATTRIBUTION_CACHE_TABLES = (
|
|
5042
|
+
"claude_account_timeline", "claude_cost_decisions",
|
|
5043
|
+
)
|
|
5044
|
+
_CLAUDE_ATTRIBUTION_CACHE_DDL = (
|
|
5045
|
+
"CREATE TABLE IF NOT EXISTS claude_account_timeline ("
|
|
5046
|
+
" captured_at_utc TEXT NOT NULL,"
|
|
5047
|
+
" precision_us INTEGER NOT NULL,"
|
|
5048
|
+
" account_key TEXT NULL,"
|
|
5049
|
+
" journal_segment TEXT NOT NULL,"
|
|
5050
|
+
" journal_offset INTEGER NOT NULL)",
|
|
5051
|
+
"CREATE INDEX IF NOT EXISTS idx_claude_account_timeline_captured "
|
|
5052
|
+
"ON claude_account_timeline(captured_at_utc)",
|
|
5053
|
+
"CREATE TABLE IF NOT EXISTS claude_cost_decisions ("
|
|
5054
|
+
" decision_key TEXT PRIMARY KEY,"
|
|
5055
|
+
" form TEXT NOT NULL,"
|
|
5056
|
+
" msg_id TEXT,"
|
|
5057
|
+
" req_id TEXT,"
|
|
5058
|
+
" loc TEXT,"
|
|
5059
|
+
" byte_offset INTEGER,"
|
|
5060
|
+
" digest TEXT,"
|
|
5061
|
+
" account_key TEXT NULL,"
|
|
5062
|
+
" basis TEXT NOT NULL,"
|
|
5063
|
+
" reason TEXT NULL,"
|
|
5064
|
+
" rule INTEGER NOT NULL,"
|
|
5065
|
+
" decided_at_utc TEXT NOT NULL,"
|
|
5066
|
+
" record_ts_utc TEXT NULL,"
|
|
5067
|
+
" projection_json TEXT NULL,"
|
|
5068
|
+
" journal_segment TEXT NOT NULL,"
|
|
5069
|
+
" journal_offset INTEGER NOT NULL)",
|
|
5070
|
+
"CREATE INDEX IF NOT EXISTS idx_claude_cost_decisions_decided "
|
|
5071
|
+
"ON claude_cost_decisions(decided_at_utc)",
|
|
5072
|
+
"CREATE INDEX IF NOT EXISTS idx_claude_cost_decisions_loc "
|
|
5073
|
+
"ON claude_cost_decisions(loc, byte_offset)",
|
|
5074
|
+
)
|
|
5075
|
+
|
|
5076
|
+
|
|
4811
5077
|
_CODEX_968_CACHE_TABLES = (
|
|
4812
5078
|
"codex_prune_debt", "codex_barrier_membership", "codex_barrier_targets",
|
|
4813
5079
|
"codex_barrier_population", "codex_barrier_certificates",
|
|
@@ -5209,6 +5475,28 @@ CACHE_REDERIVABLE_OBJECTS: "tuple[SchemaDeliveryObject, ...]" = (
|
|
|
5209
5475
|
"index", "idx_codex_window_attributions_read_order",
|
|
5210
5476
|
"_apply_codex_window_attribution_read_index",
|
|
5211
5477
|
"047_spill_free_read_paths"),
|
|
5478
|
+
# #944 spec 5.7: migration 050's handler calls the shared ensure helper.
|
|
5479
|
+
*(SchemaDeliveryObject(
|
|
5480
|
+
"table", table, "_apply_claude_attribution_schema",
|
|
5481
|
+
"050_preserve_claude_cost_attribution")
|
|
5482
|
+
for table in _CLAUDE_ATTRIBUTION_CACHE_TABLES),
|
|
5483
|
+
SchemaDeliveryObject(
|
|
5484
|
+
"column", "session_entries.attribution_decision_key",
|
|
5485
|
+
"_apply_claude_attribution_schema",
|
|
5486
|
+
"050_preserve_claude_cost_attribution"),
|
|
5487
|
+
*(SchemaDeliveryObject(
|
|
5488
|
+
"index", index, "_apply_claude_attribution_schema",
|
|
5489
|
+
"050_preserve_claude_cost_attribution")
|
|
5490
|
+
for index in ("idx_claude_account_timeline_captured",
|
|
5491
|
+
"idx_claude_cost_decisions_decided",
|
|
5492
|
+
"idx_claude_cost_decisions_loc",
|
|
5493
|
+
"idx_entries_attr_unlinked",
|
|
5494
|
+
"idx_entries_attr_nonkeyed")),
|
|
5495
|
+
# #927 spec §4.2 B1: migration 051's handler calls the shared helper.
|
|
5496
|
+
*(SchemaDeliveryObject(
|
|
5497
|
+
kind, name, "_apply_codex_retained_identity_inventory",
|
|
5498
|
+
"051_codex_retained_identities")
|
|
5499
|
+
for kind, name in _CODEX_RETAINED_IDENTITY_OBJECTS),
|
|
5212
5500
|
)
|
|
5213
5501
|
|
|
5214
5502
|
|
|
@@ -5251,6 +5539,51 @@ def _apply_codex_container_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5251
5539
|
conn.execute(statement)
|
|
5252
5540
|
|
|
5253
5541
|
|
|
5542
|
+
def _apply_claude_attribution_schema(conn: sqlite3.Connection) -> None:
|
|
5543
|
+
"""#944 spec 5.1/5.4: deliver the write-time Claude cost attribution
|
|
5544
|
+
schema -- the two derived tables, the ``session_entries`` link column and
|
|
5545
|
+
its two partial indexes.
|
|
5546
|
+
|
|
5547
|
+
Shared by ``_apply_cache_schema`` and the migration that delivers it to an
|
|
5548
|
+
already-current store, so both shapes converge. The link column goes
|
|
5549
|
+
through ``add_column_if_missing`` only, never the CREATE TABLE body
|
|
5550
|
+
(ALTER TABLE ADD COLUMN appends, so a column in both places sits at a
|
|
5551
|
+
different ordinal on a fresh store than on a migrated one -- the #195
|
|
5552
|
+
hazard). Idempotent.
|
|
5553
|
+
"""
|
|
5554
|
+
for statement in _CLAUDE_ATTRIBUTION_CACHE_DDL:
|
|
5555
|
+
conn.execute(statement)
|
|
5556
|
+
if _table_exists(conn, "session_entries"):
|
|
5557
|
+
# Literal column name keeps the schema-delivery AST guard complete.
|
|
5558
|
+
add_column_if_missing(
|
|
5559
|
+
conn, "session_entries", "attribution_decision_key", "TEXT")
|
|
5560
|
+
conn.execute(
|
|
5561
|
+
"CREATE INDEX IF NOT EXISTS idx_entries_attr_unlinked "
|
|
5562
|
+
"ON session_entries(id) WHERE attribution_decision_key IS NULL")
|
|
5563
|
+
conn.execute(
|
|
5564
|
+
"CREATE INDEX IF NOT EXISTS idx_entries_attr_nonkeyed "
|
|
5565
|
+
"ON session_entries(attribution_decision_key) "
|
|
5566
|
+
"WHERE msg_id IS NULL OR req_id IS NULL")
|
|
5567
|
+
|
|
5568
|
+
|
|
5569
|
+
def claude_attribution_schema_present(conn: sqlite3.Connection) -> bool:
|
|
5570
|
+
"""Whether this store carries the #944 attribution schema (both tables
|
|
5571
|
+
and the link column). A direct handler call on a historical
|
|
5572
|
+
``pre.sqlite`` lacks it; every production store has it, because schema
|
|
5573
|
+
delivery precedes every handler (spec 5.6)."""
|
|
5574
|
+
try:
|
|
5575
|
+
tables = {row[0] for row in conn.execute(
|
|
5576
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name IN "
|
|
5577
|
+
"('claude_account_timeline', 'claude_cost_decisions')")}
|
|
5578
|
+
if len(tables) != 2:
|
|
5579
|
+
return False
|
|
5580
|
+
columns = {row[1] for row in conn.execute(
|
|
5581
|
+
"PRAGMA table_info(session_entries)")}
|
|
5582
|
+
except sqlite3.Error:
|
|
5583
|
+
return False
|
|
5584
|
+
return "attribution_decision_key" in columns
|
|
5585
|
+
|
|
5586
|
+
|
|
5254
5587
|
def _apply_codex_conversation_container_schema(conn: sqlite3.Connection) -> None:
|
|
5255
5588
|
"""#968 spec 5.3: the conversations twin of the container evidence.
|
|
5256
5589
|
|
|
@@ -5924,6 +6257,10 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5924
6257
|
"ON session_entries(source_path, line_offset)")
|
|
5925
6258
|
except sqlite3.IntegrityError:
|
|
5926
6259
|
pass
|
|
6260
|
+
# #944: write-time Claude cost attribution (derived tables, the link
|
|
6261
|
+
# column and its partial indexes). After the session_entries column adds
|
|
6262
|
+
# and BEFORE the legacy-FTS early return, like the indexes above.
|
|
6263
|
+
_apply_claude_attribution_schema(conn)
|
|
5927
6264
|
# Existing-DB guard for the skill-content fold link (cctally-dev
|
|
5928
6265
|
# skill-content-nesting): the message-level sourceToolUseID. Idempotent
|
|
5929
6266
|
# column-add (no marker, no version); cache migration 006 then re-ingests
|
|
@@ -5995,6 +6332,8 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
5995
6332
|
_apply_codex_quota_unresolved_model_index(conn)
|
|
5996
6333
|
_apply_codex_quota_change_ledger(conn)
|
|
5997
6334
|
_apply_codex_accounting_change_ledger(conn)
|
|
6335
|
+
# #927: after cache_meta and the four families exist (spec §4.2 B1).
|
|
6336
|
+
_apply_codex_retained_identity_inventory(conn)
|
|
5998
6337
|
# #901 (spec §5.1-§5.3): the latest-capture summary, the general loader's
|
|
5999
6338
|
# ordering index, the W2 lookup indexes, the pricing model indexes and the
|
|
6000
6339
|
# window-attribution read order (Amendment 1 item 1). BEFORE the
|
|
@@ -6194,6 +6533,13 @@ def _apply_conversations_schema(conn: sqlite3.Connection) -> None:
|
|
|
6194
6533
|
-- quota_window_snapshots, which was dropped above together with its
|
|
6195
6534
|
-- triggers and seek index. Drop the table as well.
|
|
6196
6535
|
DROP TABLE IF EXISTS codex_quota_partition_latest;
|
|
6536
|
+
-- #944: the Claude cost attribution tables are cache.db accounting
|
|
6537
|
+
-- state; conversation code reads them through the cache_db attachment.
|
|
6538
|
+
DROP TABLE IF EXISTS claude_account_timeline;
|
|
6539
|
+
DROP TABLE IF EXISTS claude_cost_decisions;
|
|
6540
|
+
-- #927: the retained-identity inventory is cache.db prune state. Its
|
|
6541
|
+
-- triggers went with the four family tables dropped above.
|
|
6542
|
+
DROP TABLE IF EXISTS codex_retained_identities;
|
|
6197
6543
|
|
|
6198
6544
|
CREATE TABLE IF NOT EXISTS conversation_source_files (
|
|
6199
6545
|
path TEXT PRIMARY KEY,
|
|
@@ -8280,6 +8626,11 @@ def _001_dedup_highest_wins_locked(conn: sqlite3.Connection) -> None:
|
|
|
8280
8626
|
"corrected dedup (one-time; may take 10-30s depending on "
|
|
8281
8627
|
"JSONL volume)..."
|
|
8282
8628
|
)
|
|
8629
|
+
# #944 spec 5.6: no row is deleted before it is linked. Every unlinked
|
|
8630
|
+
# row's stamp (NULL included, whatever the population: a pending 029 is no
|
|
8631
|
+
# proof that NULLs predate account stamping) is aligned or exported first;
|
|
8632
|
+
# an unwritable journal defers 001 with nothing wiped.
|
|
8633
|
+
_link_claude_rows_before_001_wipe(conn)
|
|
8283
8634
|
# D3 — BEGIN IMMEDIATE so the destructive DELETEs are race-guarded,
|
|
8284
8635
|
# not just the marker insert. The dispatcher snapshots the applied
|
|
8285
8636
|
# set ONCE before its registry walk (``_run_pending_migrations``),
|
|
@@ -8342,6 +8693,48 @@ def _001_dedup_highest_wins_locked(conn: sqlite3.Connection) -> None:
|
|
|
8342
8693
|
raise
|
|
8343
8694
|
|
|
8344
8695
|
|
|
8696
|
+
#: #944 spec 5.7: unlinked rows one migration-050 invocation links (and the
|
|
8697
|
+
#: round size migration 001's preserve-before-wipe commits per round).
|
|
8698
|
+
_CLAUDE_PRESERVATION_CHUNK_ROWS = 50_000
|
|
8699
|
+
|
|
8700
|
+
|
|
8701
|
+
def _link_claude_rows_before_001_wipe(conn: sqlite3.Connection) -> None:
|
|
8702
|
+
"""Link every unlinked ``session_entries`` row before cache 001 wipes the
|
|
8703
|
+
table (#944 spec 5.6): replay pending decisions, then align or export in
|
|
8704
|
+
committed rounds. A store without the attribution schema (a direct handler
|
|
8705
|
+
call on a historical ``pre.sqlite``) holds no decisions and behaves
|
|
8706
|
+
exactly as before. ``MigrationGateNotMet`` when the journal cannot be read
|
|
8707
|
+
or written, so 001 stays pending with nothing wiped."""
|
|
8708
|
+
if not claude_attribution_schema_present(conn):
|
|
8709
|
+
return
|
|
8710
|
+
if conn.execute(
|
|
8711
|
+
"SELECT 1 FROM schema_migrations WHERE name = ? LIMIT 1",
|
|
8712
|
+
("001_dedup_highest_wins",),
|
|
8713
|
+
).fetchone() is not None:
|
|
8714
|
+
return # a concurrent opener already wiped; nothing left to link
|
|
8715
|
+
import _cctally_claude_attribution as G
|
|
8716
|
+
try:
|
|
8717
|
+
hw = G.rehydrate_claude_attribution(conn, authoritative=False)
|
|
8718
|
+
if hw is None:
|
|
8719
|
+
raise G.AttributionJournalUnavailable(
|
|
8720
|
+
"the journal could not be read")
|
|
8721
|
+
conn.commit()
|
|
8722
|
+
after = 0
|
|
8723
|
+
while True:
|
|
8724
|
+
result = G.preserve_unlinked_rows(
|
|
8725
|
+
conn, journal_hw=hw, after_id=after,
|
|
8726
|
+
limit=_CLAUDE_PRESERVATION_CHUNK_ROWS)
|
|
8727
|
+
conn.commit()
|
|
8728
|
+
if not result.remaining:
|
|
8729
|
+
return
|
|
8730
|
+
after = result.last_id
|
|
8731
|
+
except G.AttributionJournalUnavailable as exc:
|
|
8732
|
+
conn.rollback()
|
|
8733
|
+
raise MigrationGateNotMet(
|
|
8734
|
+
f"Claude cost attribution could not be made durable before the "
|
|
8735
|
+
f"cache 001 wipe ({exc}); deferring") from exc
|
|
8736
|
+
|
|
8737
|
+
|
|
8345
8738
|
# === Region 7c: Cache migration 002_conversation_messages_backfill ===
|
|
8346
8739
|
|
|
8347
8740
|
@cache_migration("002_conversation_messages_backfill")
|
|
@@ -9339,16 +9732,24 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
|
|
|
9339
9732
|
resolved (not pending) outcome → no-op (leave NULL). Idempotent: only touches
|
|
9340
9733
|
``account_key IS NULL`` rows. Central stamp via the dispatcher (#140); the
|
|
9341
9734
|
handler does NOT self-stamp.
|
|
9735
|
+
|
|
9736
|
+
#944 spec 5.6: a row linked to a durable Claude cost decision (a decided
|
|
9737
|
+
NULL included) is never overwritten, whatever the skip/unskip order. A
|
|
9738
|
+
store without the attribution schema (a direct call on a historical
|
|
9739
|
+
``pre.sqlite``) has no links, so the predicate is omitted there.
|
|
9342
9740
|
"""
|
|
9343
9741
|
import _cctally_journal as _jr
|
|
9344
9742
|
import _lib_accounts as _acc
|
|
9345
9743
|
|
|
9744
|
+
unlinked = (" AND attribution_decision_key IS NULL"
|
|
9745
|
+
if claude_attribution_schema_present(conn) else "")
|
|
9346
9746
|
op_value = _accounts_cutover_value_or_defer("Claude account backfill")
|
|
9347
9747
|
if op_value is None:
|
|
9348
9748
|
# Cutover hasn't appended its op yet. Defer only if there is legacy Claude
|
|
9349
9749
|
# history to attribute; a fresh install (no such rows) proceeds as a no-op.
|
|
9350
9750
|
pending = conn.execute(
|
|
9351
|
-
"SELECT 1 FROM session_entries WHERE account_key IS NULL
|
|
9751
|
+
"SELECT 1 FROM session_entries WHERE account_key IS NULL"
|
|
9752
|
+
+ unlinked + " LIMIT 1"
|
|
9352
9753
|
).fetchone()
|
|
9353
9754
|
if pending is not None:
|
|
9354
9755
|
raise MigrationGateNotMet(
|
|
@@ -9361,7 +9762,8 @@ def _029_backfill_claude_account(conn: sqlite3.Connection) -> None:
|
|
|
9361
9762
|
conn.execute("BEGIN IMMEDIATE")
|
|
9362
9763
|
try:
|
|
9363
9764
|
conn.execute(
|
|
9364
|
-
"UPDATE session_entries SET account_key=? WHERE account_key IS NULL"
|
|
9765
|
+
"UPDATE session_entries SET account_key=? WHERE account_key IS NULL"
|
|
9766
|
+
+ unlinked,
|
|
9365
9767
|
(op_value,),
|
|
9366
9768
|
)
|
|
9367
9769
|
conn.execute(
|
|
@@ -10268,6 +10670,144 @@ def _049_codex_rollout_containers(conn: sqlite3.Connection) -> None:
|
|
|
10268
10670
|
conn.commit()
|
|
10269
10671
|
|
|
10270
10672
|
|
|
10673
|
+
@cache_migration("050_preserve_claude_cost_attribution")
|
|
10674
|
+
def _050_preserve_claude_cost_attribution(conn: sqlite3.Connection) -> None:
|
|
10675
|
+
"""#944 spec 5.7 (U1): make every existing Claude cost stamp durable.
|
|
10676
|
+
|
|
10677
|
+
Delivers the attribution schema (the shared helper every fresh store
|
|
10678
|
+
gets through ``_apply_cache_schema``), then links up to
|
|
10679
|
+
``_CLAUDE_PRESERVATION_CHUNK_ROWS`` unlinked ``session_entries`` rows per
|
|
10680
|
+
invocation, in ``id`` order past the recorded cursor: a row whose
|
|
10681
|
+
identity already holds a decision is aligned to it, every other row's
|
|
10682
|
+
CURRENT stamp (NULL included, orphan sources included) is exported as a
|
|
10683
|
+
``preserved_history`` decision -- keyed form, or the legacy projection
|
|
10684
|
+
form for a row lacking an id. Never consults the kernel and never reads
|
|
10685
|
+
a transcript.
|
|
10686
|
+
|
|
10687
|
+
Gate: migration 029 applied or skipped, because 029 backfills NULL stamps
|
|
10688
|
+
U1 then preserves as they stand. Each chunk appends (fsync) and then, in
|
|
10689
|
+
one transaction, inserts the decisions, sets the links and records
|
|
10690
|
+
``claude_cost_preservation``; while an unlinked row remains past the
|
|
10691
|
+
cursor the handler raises ``MigrationGateNotMet`` so the migration stays
|
|
10692
|
+
pending with its committed chunks kept. An unwritable journal records
|
|
10693
|
+
``lastError``/``lastErrorAt`` in a separate small transaction (doctor's
|
|
10694
|
+
stall WARN) and defers; nothing is wiped and nothing is stamped.
|
|
10695
|
+
|
|
10696
|
+
Takes the cache writer flock on a direct call (the dispatcher already
|
|
10697
|
+
holds it). Re-running is a no-op once every row is linked. NO
|
|
10698
|
+
self-stamp -- the dispatcher central-stamps on a clean return (#140).
|
|
10699
|
+
"""
|
|
10700
|
+
_apply_claude_attribution_schema(conn)
|
|
10701
|
+
conn.commit()
|
|
10702
|
+
gate = conn.execute(
|
|
10703
|
+
"SELECT 1 FROM schema_migrations WHERE name = ? UNION ALL "
|
|
10704
|
+
"SELECT 1 FROM schema_migrations_skipped WHERE name = ? LIMIT 1",
|
|
10705
|
+
("029_backfill_claude_account", "029_backfill_claude_account"),
|
|
10706
|
+
).fetchone() if _table_exists(conn, "schema_migrations_skipped") else (
|
|
10707
|
+
conn.execute(
|
|
10708
|
+
"SELECT 1 FROM schema_migrations WHERE name = ? LIMIT 1",
|
|
10709
|
+
("029_backfill_claude_account",)).fetchone())
|
|
10710
|
+
if gate is None:
|
|
10711
|
+
raise MigrationGateNotMet(
|
|
10712
|
+
"cache 029 is neither applied nor skipped; deferring the Claude "
|
|
10713
|
+
"cost attribution export until its NULL backfill has run")
|
|
10714
|
+
lock_path = (
|
|
10715
|
+
None
|
|
10716
|
+
if _CACHE_DISPATCH_WRITER_LOCK_HELD.get()
|
|
10717
|
+
else _cache_db_lock_path_for_conn(conn)
|
|
10718
|
+
)
|
|
10719
|
+
lock_fh = None
|
|
10720
|
+
if lock_path is not None:
|
|
10721
|
+
lock_fh = open(lock_path, "w")
|
|
10722
|
+
try:
|
|
10723
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
10724
|
+
except BlockingIOError:
|
|
10725
|
+
lock_fh.close()
|
|
10726
|
+
raise MigrationGateNotMet(
|
|
10727
|
+
"cache.db.lock held by a concurrent sync_cache; deferring "
|
|
10728
|
+
"cache 050 Claude cost attribution export (#944)")
|
|
10729
|
+
try:
|
|
10730
|
+
_050_export_one_chunk(conn)
|
|
10731
|
+
finally:
|
|
10732
|
+
if lock_fh is not None:
|
|
10733
|
+
try:
|
|
10734
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
10735
|
+
except OSError:
|
|
10736
|
+
pass
|
|
10737
|
+
lock_fh.close()
|
|
10738
|
+
|
|
10739
|
+
|
|
10740
|
+
def _050_export_one_chunk(conn: sqlite3.Connection) -> None:
|
|
10741
|
+
import _cctally_claude_attribution as G
|
|
10742
|
+
|
|
10743
|
+
def unlinked_count() -> int:
|
|
10744
|
+
return conn.execute(
|
|
10745
|
+
"SELECT COUNT(*) FROM session_entries "
|
|
10746
|
+
"WHERE attribution_decision_key IS NULL").fetchone()[0]
|
|
10747
|
+
|
|
10748
|
+
progress = G.read_preservation(conn)
|
|
10749
|
+
if progress is None:
|
|
10750
|
+
progress = {"afterId": 0, "saved": 0, "total": unlinked_count(),
|
|
10751
|
+
"lastError": None, "lastErrorAt": None,
|
|
10752
|
+
"lastChunkAt": None}
|
|
10753
|
+
after = int(progress.get("afterId") or 0)
|
|
10754
|
+
try:
|
|
10755
|
+
hw = G.rehydrate_claude_attribution(conn, authoritative=False)
|
|
10756
|
+
if hw is None:
|
|
10757
|
+
raise G.AttributionJournalUnavailable(
|
|
10758
|
+
"the journal could not be read")
|
|
10759
|
+
conn.commit()
|
|
10760
|
+
result = G.preserve_unlinked_rows(
|
|
10761
|
+
conn, journal_hw=hw, after_id=after,
|
|
10762
|
+
limit=_CLAUDE_PRESERVATION_CHUNK_ROWS)
|
|
10763
|
+
except G.AttributionJournalUnavailable as exc:
|
|
10764
|
+
conn.rollback()
|
|
10765
|
+
if not exc.contended:
|
|
10766
|
+
progress["lastError"] = str(exc)
|
|
10767
|
+
progress["lastErrorAt"] = now_utc_iso()
|
|
10768
|
+
G.write_preservation(conn, progress)
|
|
10769
|
+
conn.commit()
|
|
10770
|
+
raise MigrationGateNotMet(
|
|
10771
|
+
f"Claude cost attribution export deferred: {exc}") from exc
|
|
10772
|
+
total = int(progress.get("total") or 0)
|
|
10773
|
+
progress.update({
|
|
10774
|
+
"afterId": max(after, result.last_id),
|
|
10775
|
+
# Aligned rows and rows ingest linked meanwhile count as saved.
|
|
10776
|
+
"saved": max(0, min(total, total - unlinked_count())),
|
|
10777
|
+
"lastError": None, "lastErrorAt": None,
|
|
10778
|
+
"lastChunkAt": now_utc_iso(),
|
|
10779
|
+
})
|
|
10780
|
+
G.write_preservation(conn, progress)
|
|
10781
|
+
conn.commit()
|
|
10782
|
+
if result.remaining:
|
|
10783
|
+
raise MigrationGateNotMet(
|
|
10784
|
+
f"Claude cost attribution export saved {progress['saved']} of "
|
|
10785
|
+
f"{total} rows; resuming on the next open")
|
|
10786
|
+
|
|
10787
|
+
|
|
10788
|
+
@cache_migration("051_codex_retained_identities")
|
|
10789
|
+
def _051_codex_retained_identities(conn: sqlite3.Connection) -> None:
|
|
10790
|
+
"""#927 spec §4.2 B1: deliver the retained-identity inventory.
|
|
10791
|
+
|
|
10792
|
+
The schema apply that declares the table, its expression unique index and
|
|
10793
|
+
the eight INSERT/identity-UPDATE triggers is version-gated, so only a
|
|
10794
|
+
registered migration reaches an already-current store (the reason 046
|
|
10795
|
+
gives). The shared helper creates the triggers first and then records
|
|
10796
|
+
``ready`` when the four families hold no Codex row, else
|
|
10797
|
+
``bootstrap:files:0``: an existing history is backfilled off the hook by
|
|
10798
|
+
``_codex-inventory-bootstrap`` (or inline by ``cache-sync``), never here,
|
|
10799
|
+
so an ordinary open pays no history scan.
|
|
10800
|
+
|
|
10801
|
+
Takes no provider flock: the helper's IMMEDIATE transaction makes the
|
|
10802
|
+
trigger creation and the state decision atomic against every writer.
|
|
10803
|
+
Re-running is a no-op once all ten objects exist (the recorded state, and
|
|
10804
|
+
any bootstrap progress, stand). NO self-stamp — the dispatcher
|
|
10805
|
+
central-stamps on a clean return (#140).
|
|
10806
|
+
"""
|
|
10807
|
+
_apply_codex_retained_identity_inventory(conn)
|
|
10808
|
+
conn.commit()
|
|
10809
|
+
|
|
10810
|
+
|
|
10271
10811
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
10272
10812
|
|
|
10273
10813
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
|
@@ -11342,15 +11882,30 @@ def cmd_db_status(args: argparse.Namespace) -> int:
|
|
|
11342
11882
|
"migrations": [],
|
|
11343
11883
|
"_open_error": str(exc),
|
|
11344
11884
|
}
|
|
11885
|
+
# #1000 §5.10: the conversations leg opens under the diagnostic read
|
|
11886
|
+
# admission and DECLINES while the probe, recovery or another maintenance
|
|
11887
|
+
# holder owns the family; report that the same way.
|
|
11888
|
+
try:
|
|
11889
|
+
conversations_status = _db_status_for(
|
|
11890
|
+
_cctally_core.CONVERSATIONS_DB_PATH, _CONVERSATIONS_MIGRATIONS,
|
|
11891
|
+
"conversations.db",
|
|
11892
|
+
)
|
|
11893
|
+
# An admission whose lock file cannot be opened (an unwritable data
|
|
11894
|
+
# folder) is a decline as well, never a traceback.
|
|
11895
|
+
except (sqlite3.DatabaseError, OSError) as exc:
|
|
11896
|
+
conversations_status = {
|
|
11897
|
+
"path": str(_cctally_core.CONVERSATIONS_DB_PATH),
|
|
11898
|
+
"user_version": 0,
|
|
11899
|
+
"registry_size": len(_CONVERSATIONS_MIGRATIONS),
|
|
11900
|
+
"migrations": [],
|
|
11901
|
+
"_open_error": str(exc),
|
|
11902
|
+
}
|
|
11345
11903
|
payload = {
|
|
11346
11904
|
"schema_version": 1,
|
|
11347
11905
|
"databases": {
|
|
11348
11906
|
"stats.db": stats_status,
|
|
11349
11907
|
"cache.db": _db_status_for(_cctally_core.CACHE_DB_PATH, _CACHE_MIGRATIONS, "cache.db"),
|
|
11350
|
-
"conversations.db":
|
|
11351
|
-
_cctally_core.CONVERSATIONS_DB_PATH, _CONVERSATIONS_MIGRATIONS,
|
|
11352
|
-
"conversations.db",
|
|
11353
|
-
),
|
|
11908
|
+
"conversations.db": conversations_status,
|
|
11354
11909
|
},
|
|
11355
11910
|
}
|
|
11356
11911
|
# #997 D10: additive — each detached rebuild worker's state and its
|
|
@@ -11428,15 +11983,7 @@ def _db_status_for(
|
|
|
11428
11983
|
except sqlite3.OperationalError.
|
|
11429
11984
|
"""
|
|
11430
11985
|
if not db_path.exists():
|
|
11431
|
-
return
|
|
11432
|
-
"path": str(db_path),
|
|
11433
|
-
"user_version": 0,
|
|
11434
|
-
"registry_size": len(registry),
|
|
11435
|
-
"migrations": [
|
|
11436
|
-
{"seq": m.seq, "name": m.name, "status": "pending"}
|
|
11437
|
-
for m in registry
|
|
11438
|
-
],
|
|
11439
|
-
}
|
|
11986
|
+
return _db_status_absent(db_path, registry)
|
|
11440
11987
|
if db_path == _cctally_core.DB_PATH:
|
|
11441
11988
|
# #386: participate in the replacement protocol, and connect `mode=rw`
|
|
11442
11989
|
# so this diagnostic can never MATERIALISE the very stats.db a
|
|
@@ -11452,34 +11999,77 @@ def _db_status_for(
|
|
|
11452
11999
|
f"file:{p}?mode=rw", uri=True, **kw),
|
|
11453
12000
|
recover_interruptions=recover_interrupted_stats,
|
|
11454
12001
|
)
|
|
12002
|
+
elif db_label == "conversations.db":
|
|
12003
|
+
# #1000 §5.10: the diagnostic read admission (maintenance SHARED for
|
|
12004
|
+
# the connection's life, `mode=ro`), so this diagnostic can neither
|
|
12005
|
+
# open the family under the pre-rebuild probe's in-place read nor
|
|
12006
|
+
# recreate a main file a quarantine just renamed: the `exists()`
|
|
12007
|
+
# check above is a TOCTOU, and a plain `sqlite3.connect` recreates a
|
|
12008
|
+
# file removed in the gap. A decline raises `sqlite3.DatabaseError`
|
|
12009
|
+
# with the readers' existing text, which `cmd_db_status` renders as
|
|
12010
|
+
# this leg's `_open_error`.
|
|
12011
|
+
import _cctally_cache
|
|
12012
|
+
|
|
12013
|
+
with _cctally_cache.conversation_ro_guarded(timeout=5.0) as conn:
|
|
12014
|
+
if conn is None:
|
|
12015
|
+
if not os.path.lexists(db_path):
|
|
12016
|
+
return _db_status_absent(db_path, registry)
|
|
12017
|
+
raise sqlite3.DatabaseError(
|
|
12018
|
+
_cctally_cache.conversation_ro_decline_reason())
|
|
12019
|
+
return _db_status_rows(conn, db_path, registry, db_label)
|
|
11455
12020
|
else:
|
|
11456
12021
|
conn = sqlite3.connect(db_path)
|
|
11457
12022
|
try:
|
|
11458
|
-
|
|
11459
|
-
# Tolerate missing tables (e.g., cache.db never opened by framework).
|
|
11460
|
-
# Alias legacy unprefixed names to their NNN_ equivalents so a
|
|
11461
|
-
# `db status` that runs before any open_db()-via-bootstrap rename
|
|
11462
|
-
# still reports legacy-marker DBs as applied, not pending.
|
|
11463
|
-
aliases = _LEGACY_MARKER_ALIASES_BY_DB.get(db_label, {})
|
|
11464
|
-
applied_rows: dict[str, str] = {}
|
|
11465
|
-
try:
|
|
11466
|
-
for row in conn.execute(
|
|
11467
|
-
"SELECT name, applied_at_utc FROM schema_migrations"
|
|
11468
|
-
).fetchall():
|
|
11469
|
-
applied_rows[aliases.get(row[0], row[0])] = row[1]
|
|
11470
|
-
except sqlite3.OperationalError:
|
|
11471
|
-
pass
|
|
11472
|
-
skipped_rows: dict[str, tuple[str, str | None]] = {}
|
|
11473
|
-
try:
|
|
11474
|
-
for row in conn.execute(
|
|
11475
|
-
"SELECT name, skipped_at_utc, reason FROM schema_migrations_skipped"
|
|
11476
|
-
).fetchall():
|
|
11477
|
-
skipped_rows[row[0]] = (row[1], row[2])
|
|
11478
|
-
except sqlite3.OperationalError:
|
|
11479
|
-
pass
|
|
12023
|
+
return _db_status_rows(conn, db_path, registry, db_label)
|
|
11480
12024
|
finally:
|
|
11481
12025
|
conn.close()
|
|
11482
12026
|
|
|
12027
|
+
|
|
12028
|
+
def _db_status_absent(
|
|
12029
|
+
db_path: pathlib.Path, registry: list[Migration],
|
|
12030
|
+
) -> dict:
|
|
12031
|
+
"""The status dict for an absent store: every migration pending."""
|
|
12032
|
+
return {
|
|
12033
|
+
"path": str(db_path),
|
|
12034
|
+
"user_version": 0,
|
|
12035
|
+
"registry_size": len(registry),
|
|
12036
|
+
"migrations": [
|
|
12037
|
+
{"seq": m.seq, "name": m.name, "status": "pending"}
|
|
12038
|
+
for m in registry
|
|
12039
|
+
],
|
|
12040
|
+
}
|
|
12041
|
+
|
|
12042
|
+
|
|
12043
|
+
def _db_status_rows(
|
|
12044
|
+
conn: sqlite3.Connection,
|
|
12045
|
+
db_path: pathlib.Path,
|
|
12046
|
+
registry: list[Migration],
|
|
12047
|
+
db_label: str,
|
|
12048
|
+
) -> dict:
|
|
12049
|
+
"""The status dict for ``db_path`` read through the open ``conn``."""
|
|
12050
|
+
user_version = conn.execute("PRAGMA user_version").fetchone()[0]
|
|
12051
|
+
# Tolerate missing tables (e.g., cache.db never opened by framework).
|
|
12052
|
+
# Alias legacy unprefixed names to their NNN_ equivalents so a
|
|
12053
|
+
# `db status` that runs before any open_db()-via-bootstrap rename
|
|
12054
|
+
# still reports legacy-marker DBs as applied, not pending.
|
|
12055
|
+
aliases = _LEGACY_MARKER_ALIASES_BY_DB.get(db_label, {})
|
|
12056
|
+
applied_rows: dict[str, str] = {}
|
|
12057
|
+
try:
|
|
12058
|
+
for row in conn.execute(
|
|
12059
|
+
"SELECT name, applied_at_utc FROM schema_migrations"
|
|
12060
|
+
).fetchall():
|
|
12061
|
+
applied_rows[aliases.get(row[0], row[0])] = row[1]
|
|
12062
|
+
except sqlite3.OperationalError:
|
|
12063
|
+
pass
|
|
12064
|
+
skipped_rows: dict[str, tuple[str, str | None]] = {}
|
|
12065
|
+
try:
|
|
12066
|
+
for row in conn.execute(
|
|
12067
|
+
"SELECT name, skipped_at_utc, reason FROM schema_migrations_skipped"
|
|
12068
|
+
).fetchall():
|
|
12069
|
+
skipped_rows[row[0]] = (row[1], row[2])
|
|
12070
|
+
except sqlite3.OperationalError:
|
|
12071
|
+
pass
|
|
12072
|
+
|
|
11483
12073
|
failed_names = _db_status_failed_names_from_log(db_label)
|
|
11484
12074
|
migrations_out: list[dict] = []
|
|
11485
12075
|
for m in registry:
|
|
@@ -11631,6 +12221,12 @@ def _acquire_db_admin_writer_flocks(
|
|
|
11631
12221
|
``PRAGMA user_version = 0`` against the live stats index. stats.db now takes
|
|
11632
12222
|
its maintenance lock EXCLUSIVE, which is the regime spec section 3.1 puts
|
|
11633
12223
|
administrative mutation in.
|
|
12224
|
+
|
|
12225
|
+
#1000 §5.8: conversations.db takes, in the conversation-writer lock order
|
|
12226
|
+
(maintenance before the providers, Claude then Codex), its maintenance
|
|
12227
|
+
lock SHARED and both provider locks EXCLUSIVE, as every conversation
|
|
12228
|
+
writer does, so `db skip` / `db unskip` can neither write the store under
|
|
12229
|
+
the pre-rebuild probe's in-place read nor across recovery's quarantine.
|
|
11634
12230
|
"""
|
|
11635
12231
|
from _lib_cache_writer_lock import acquire_ordered_flocks
|
|
11636
12232
|
|
|
@@ -11650,6 +12246,16 @@ def _acquire_db_admin_writer_flocks(
|
|
|
11650
12246
|
if held is not None:
|
|
11651
12247
|
_cctally_core.note_stats_maintenance_acquired()
|
|
11652
12248
|
return held
|
|
12249
|
+
if db_label == "conversations.db":
|
|
12250
|
+
return acquire_ordered_flocks(
|
|
12251
|
+
[
|
|
12252
|
+
(_cctally_core.CONVERSATIONS_LOCK_MAINTENANCE_PATH,
|
|
12253
|
+
fcntl.LOCK_SH),
|
|
12254
|
+
(_cctally_core.CONVERSATIONS_LOCK_PATH, fcntl.LOCK_EX),
|
|
12255
|
+
(_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, fcntl.LOCK_EX),
|
|
12256
|
+
],
|
|
12257
|
+
timeout=timeout,
|
|
12258
|
+
)
|
|
11653
12259
|
return []
|
|
11654
12260
|
|
|
11655
12261
|
|
|
@@ -11976,11 +12582,49 @@ def _encode_repair_owner(
|
|
|
11976
12582
|
) + "\n"
|
|
11977
12583
|
|
|
11978
12584
|
|
|
11979
|
-
|
|
12585
|
+
#: #1020: the record file inside a directory-form owner record, and how many
|
|
12586
|
+
#: times a reader resolves and reads it before reporting it unreadable.
|
|
12587
|
+
_OWNER_RECORD_FILE_NAME = "owner"
|
|
12588
|
+
_OWNER_RECORD_READ_ATTEMPTS = 3
|
|
12589
|
+
|
|
12590
|
+
|
|
12591
|
+
def _owner_record_file(marker: pathlib.Path) -> pathlib.Path:
|
|
12592
|
+
"""Return the file that holds ``marker``'s owner record (#1020).
|
|
12593
|
+
|
|
12594
|
+
A marker published where hard links work is the record itself; one
|
|
12595
|
+
published on a volume without them is a directory holding ``owner``. A
|
|
12596
|
+
failed ``lstat`` resolves to the marker, so the read fails as before."""
|
|
11980
12597
|
try:
|
|
11981
|
-
|
|
12598
|
+
st = os.lstat(marker)
|
|
11982
12599
|
except OSError:
|
|
11983
|
-
return
|
|
12600
|
+
return marker
|
|
12601
|
+
if stat.S_ISDIR(st.st_mode):
|
|
12602
|
+
return marker / _OWNER_RECORD_FILE_NAME
|
|
12603
|
+
return marker
|
|
12604
|
+
|
|
12605
|
+
|
|
12606
|
+
def _read_owner_record_text(
|
|
12607
|
+
marker: pathlib.Path,
|
|
12608
|
+
) -> "tuple[str, pathlib.Path] | None":
|
|
12609
|
+
"""Read ``marker``'s owner record; return its stripped text and the file
|
|
12610
|
+
read, or ``None`` when it cannot be read.
|
|
12611
|
+
|
|
12612
|
+
A read that finds the other form (``IsADirectoryError`` /
|
|
12613
|
+
``NotADirectoryError``) means one owner released the record and another
|
|
12614
|
+
published it in the other form between the ``lstat`` and the read, so the
|
|
12615
|
+
record is resolved and read again, a bounded number of times."""
|
|
12616
|
+
for _attempt in range(_OWNER_RECORD_READ_ATTEMPTS):
|
|
12617
|
+
record = _owner_record_file(marker)
|
|
12618
|
+
try:
|
|
12619
|
+
return record.read_text().strip(), record
|
|
12620
|
+
except (IsADirectoryError, NotADirectoryError):
|
|
12621
|
+
continue
|
|
12622
|
+
except OSError:
|
|
12623
|
+
return None
|
|
12624
|
+
return None
|
|
12625
|
+
|
|
12626
|
+
|
|
12627
|
+
def _decode_repair_owner(raw: str) -> "dict[str, Any] | None":
|
|
11984
12628
|
def legacy_owner() -> "dict[str, Any] | None":
|
|
11985
12629
|
# Legacy markers contained only a PID. They remain live for a bounded
|
|
11986
12630
|
# lease when that PID exists, preventing a reused PID from wedging the
|
|
@@ -12012,8 +12656,22 @@ def _read_repair_owner(marker: pathlib.Path) -> "dict[str, Any] | None":
|
|
|
12012
12656
|
return decoded
|
|
12013
12657
|
|
|
12014
12658
|
|
|
12659
|
+
def _read_repair_owner_record(
|
|
12660
|
+
marker: pathlib.Path,
|
|
12661
|
+
) -> "tuple[dict[str, Any] | None, pathlib.Path | None]":
|
|
12662
|
+
read = _read_owner_record_text(marker)
|
|
12663
|
+
if read is None:
|
|
12664
|
+
return None, None
|
|
12665
|
+
raw, record = read
|
|
12666
|
+
return _decode_repair_owner(raw), record
|
|
12667
|
+
|
|
12668
|
+
|
|
12669
|
+
def _read_repair_owner(marker: pathlib.Path) -> "dict[str, Any] | None":
|
|
12670
|
+
return _read_repair_owner_record(marker)[0]
|
|
12671
|
+
|
|
12672
|
+
|
|
12015
12673
|
def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
|
|
12016
|
-
owner =
|
|
12674
|
+
owner, record = _read_repair_owner_record(marker)
|
|
12017
12675
|
if owner is None:
|
|
12018
12676
|
return False, "malformed owner record"
|
|
12019
12677
|
pid = int(owner["pid"])
|
|
@@ -12026,8 +12684,11 @@ def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
|
|
|
12026
12684
|
if actual_start == expected_start:
|
|
12027
12685
|
return True, f"pid {pid}"
|
|
12028
12686
|
return False, f"pid {pid} was reused"
|
|
12687
|
+
# #1020: the lease is the age of the record file that was read, never a
|
|
12688
|
+
# directory marker's own mtime (an AppleDouble companion moves it); a
|
|
12689
|
+
# record file gone since the read was released and is not live.
|
|
12029
12690
|
try:
|
|
12030
|
-
age_seconds = max(0.0, time.time() -
|
|
12691
|
+
age_seconds = max(0.0, time.time() - record.stat().st_mtime)
|
|
12031
12692
|
except OSError:
|
|
12032
12693
|
return False, "owner record disappeared"
|
|
12033
12694
|
if age_seconds <= _REPAIR_OWNER_UNKNOWN_IDENTITY_LEASE_SECONDS:
|
|
@@ -12035,21 +12696,94 @@ def _repair_marker_is_live(marker: pathlib.Path) -> "tuple[bool, str]":
|
|
|
12035
12696
|
return False, f"pid {pid} owner identity lease expired"
|
|
12036
12697
|
|
|
12037
12698
|
|
|
12699
|
+
def _remove_owner_record(marker: pathlib.Path) -> None:
|
|
12700
|
+
"""Remove ``marker``'s owner record in either form (#1020).
|
|
12701
|
+
|
|
12702
|
+
A file (by ``lstat``, so a symlink is never followed) is unlinked. A
|
|
12703
|
+
directory is renamed to a unique private tomb, the one step readers
|
|
12704
|
+
observe; the parent is fsynced; only then is the tomb deleted, best effort.
|
|
12705
|
+
An already-absent marker is not an error. An error at the parent fsync
|
|
12706
|
+
propagates after the name is gone and leaves the tomb, so cleanup never
|
|
12707
|
+
precedes the barrier."""
|
|
12708
|
+
try:
|
|
12709
|
+
st = os.lstat(marker)
|
|
12710
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
12711
|
+
return
|
|
12712
|
+
tomb = None
|
|
12713
|
+
try:
|
|
12714
|
+
if stat.S_ISDIR(st.st_mode):
|
|
12715
|
+
tomb = marker.with_name(
|
|
12716
|
+
f".{marker.name}.{os.getpid()}-{time.time_ns()}-"
|
|
12717
|
+
f"{os.urandom(4).hex()}.removed"
|
|
12718
|
+
)
|
|
12719
|
+
os.rename(marker, tomb)
|
|
12720
|
+
else:
|
|
12721
|
+
os.unlink(marker)
|
|
12722
|
+
except FileNotFoundError:
|
|
12723
|
+
return
|
|
12724
|
+
_fsync_directory(marker.parent)
|
|
12725
|
+
if tomb is not None:
|
|
12726
|
+
shutil.rmtree(tomb, ignore_errors=True)
|
|
12727
|
+
|
|
12728
|
+
|
|
12729
|
+
def _open_owner_record_lock_fd(marker: pathlib.Path) -> int:
|
|
12730
|
+
"""Open (creating) ``<marker>.lock``, the removal lock's file (#1020),
|
|
12731
|
+
with the maintenance-lock idiom of
|
|
12732
|
+
``_cctally_cache._open_maintenance_lock_fd``; the caller owns the fd."""
|
|
12733
|
+
lock_path = marker.with_name(f"{marker.name}.lock")
|
|
12734
|
+
flags = os.O_RDWR | os.O_CREAT | os.O_APPEND | getattr(os, "O_CLOEXEC", 0)
|
|
12735
|
+
try:
|
|
12736
|
+
return os.open(lock_path, flags, 0o666)
|
|
12737
|
+
except FileNotFoundError:
|
|
12738
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
12739
|
+
return os.open(lock_path, flags, 0o666)
|
|
12740
|
+
|
|
12741
|
+
|
|
12742
|
+
@contextlib.contextmanager
|
|
12743
|
+
def _owner_record_removal_lock(marker: pathlib.Path) -> "Iterator[None]":
|
|
12744
|
+
"""Hold ``<marker>.lock`` exclusive for one removal of ``marker`` (#1020).
|
|
12745
|
+
|
|
12746
|
+
Every remover (both stale reclaimers, both releases) holds it across its
|
|
12747
|
+
absence check, read, judgment and removal, so the record it judged is the
|
|
12748
|
+
one it removes: while a record is present only a removal (which waits
|
|
12749
|
+
here) or nothing can change it, because publication needs the name
|
|
12750
|
+
absent. Publication never takes it. It is a leaf: nothing is acquired
|
|
12751
|
+
under it, and no caller holding it calls anything that takes it again
|
|
12752
|
+
(``flock`` conflicts between two descriptors of one process). Opened with
|
|
12753
|
+
the maintenance-lock idiom of ``_cctally_cache._open_maintenance_lock_fd``,
|
|
12754
|
+
on its own descriptor per acquisition."""
|
|
12755
|
+
fd = _open_owner_record_lock_fd(marker)
|
|
12756
|
+
try:
|
|
12757
|
+
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
12758
|
+
yield
|
|
12759
|
+
finally:
|
|
12760
|
+
os.close(fd)
|
|
12761
|
+
|
|
12762
|
+
|
|
12763
|
+
def _owner_record_absent(marker: pathlib.Path) -> bool:
|
|
12764
|
+
"""Whether ``marker``'s name is absent, by ``lstat`` (a dangling symlink
|
|
12765
|
+
is present). Only ``FileNotFoundError`` / ``NotADirectoryError`` prove
|
|
12766
|
+
absence; every other lookup error propagates, never read as absence."""
|
|
12767
|
+
try:
|
|
12768
|
+
os.lstat(marker)
|
|
12769
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
12770
|
+
return True
|
|
12771
|
+
return False
|
|
12772
|
+
|
|
12773
|
+
|
|
12038
12774
|
def _remove_stale_repair_marker(path: pathlib.Path) -> "tuple[bool, str]":
|
|
12039
12775
|
"""Remove a provably stale marker; the caller owns maintenance-exclusive."""
|
|
12040
12776
|
marker = _repair_marker_path(path)
|
|
12041
|
-
if not marker.exists():
|
|
12042
|
-
return True, ""
|
|
12043
|
-
live, reason = _repair_marker_is_live(marker)
|
|
12044
|
-
if live:
|
|
12045
|
-
return False, f"another {path.name} repair owns {marker} ({reason})"
|
|
12046
12777
|
try:
|
|
12047
|
-
marker
|
|
12048
|
-
|
|
12049
|
-
|
|
12778
|
+
with _owner_record_removal_lock(marker):
|
|
12779
|
+
if _owner_record_absent(marker):
|
|
12780
|
+
return True, ""
|
|
12781
|
+
live, reason = _repair_marker_is_live(marker)
|
|
12782
|
+
if live:
|
|
12783
|
+
return False, f"another {path.name} repair owns {marker} ({reason})"
|
|
12784
|
+
_remove_owner_record(marker)
|
|
12050
12785
|
except OSError as exc:
|
|
12051
12786
|
return False, f"could not remove stale repair marker {marker}: {exc}"
|
|
12052
|
-
_fsync_directory(marker.parent)
|
|
12053
12787
|
return True, reason
|
|
12054
12788
|
|
|
12055
12789
|
|
|
@@ -12065,6 +12799,96 @@ def _claim_repair_marker(
|
|
|
12065
12799
|
)
|
|
12066
12800
|
|
|
12067
12801
|
|
|
12802
|
+
def _publish_owner_record_link(temp: pathlib.Path, marker: pathlib.Path) -> None:
|
|
12803
|
+
"""Hard-link the private record ``temp`` onto ``marker`` (#1020).
|
|
12804
|
+
|
|
12805
|
+
While ``PYTEST_CURRENT_TEST`` is set, ``CCTALLY_TEST_OWNER_RECORD_LINK_ERRNO``
|
|
12806
|
+
names an errno the link raises instead, so a child-process test can stand
|
|
12807
|
+
in for a volume without hard links through the same classification a real
|
|
12808
|
+
``link(2)`` refusal takes. Production never reads it."""
|
|
12809
|
+
if os.environ.get("PYTEST_CURRENT_TEST"):
|
|
12810
|
+
name = os.environ.get("CCTALLY_TEST_OWNER_RECORD_LINK_ERRNO")
|
|
12811
|
+
if name:
|
|
12812
|
+
code = getattr(errno, name)
|
|
12813
|
+
raise OSError(code, os.strerror(code), str(temp), None, str(marker))
|
|
12814
|
+
os.link(temp, marker)
|
|
12815
|
+
|
|
12816
|
+
|
|
12817
|
+
def _trace_owner_record_publication(marker: pathlib.Path, form: str) -> None:
|
|
12818
|
+
"""Test-only (#1020): record which form (``file`` or ``directory``)
|
|
12819
|
+
published ``marker``, as ``<marker name> <form>`` lines appended to the
|
|
12820
|
+
file ``CCTALLY_TEST_OWNER_RECORD_TRACE`` names, only while
|
|
12821
|
+
``PYTEST_CURRENT_TEST`` is set."""
|
|
12822
|
+
if not os.environ.get("PYTEST_CURRENT_TEST"):
|
|
12823
|
+
return
|
|
12824
|
+
trace = os.environ.get("CCTALLY_TEST_OWNER_RECORD_TRACE")
|
|
12825
|
+
if not trace:
|
|
12826
|
+
return
|
|
12827
|
+
with open(trace, "a", encoding="utf-8") as fh:
|
|
12828
|
+
fh.write(f"{marker.name} {form}\n")
|
|
12829
|
+
|
|
12830
|
+
|
|
12831
|
+
#: #1020: a directory-form publication whose rename finds the marker name
|
|
12832
|
+
#: taken — a populated directory (``EEXIST`` / ``ENOTEMPTY``) or a file
|
|
12833
|
+
#: (``ENOTDIR``) — is contention, handled as ``FileExistsError`` on a link.
|
|
12834
|
+
_OWNER_RECORD_RENAME_CONTENTION = frozenset({
|
|
12835
|
+
errno.EEXIST, errno.ENOTEMPTY, errno.ENOTDIR,
|
|
12836
|
+
})
|
|
12837
|
+
|
|
12838
|
+
|
|
12839
|
+
def _owner_record_link_unsupported(exc: OSError) -> bool:
|
|
12840
|
+
"""Whether the publishing link failed because the volume has no hard
|
|
12841
|
+
links (#1020): ``ENOTSUP`` / ``EOPNOTSUPP`` anywhere, and ``EPERM`` on
|
|
12842
|
+
Linux, where ``link(2)`` documents it for filesystems without hard links
|
|
12843
|
+
(vfat, exfat). Every other errno, macOS ``EPERM`` included, is a real
|
|
12844
|
+
failure."""
|
|
12845
|
+
if exc.errno in (errno.ENOTSUP, errno.EOPNOTSUPP):
|
|
12846
|
+
return True
|
|
12847
|
+
return exc.errno == errno.EPERM and sys.platform.startswith("linux")
|
|
12848
|
+
|
|
12849
|
+
|
|
12850
|
+
def _publish_owner_record_directory(
|
|
12851
|
+
marker: pathlib.Path, payload: bytes, claim_id: str,
|
|
12852
|
+
) -> bool:
|
|
12853
|
+
"""Publish ``payload`` at ``marker`` as a directory holding ``owner``
|
|
12854
|
+
(#1020), for a volume without hard links; ``False`` when the name is taken.
|
|
12855
|
+
|
|
12856
|
+
The record is written and fsynced inside a private directory, which is
|
|
12857
|
+
fsynced and renamed onto the marker: rename(2) never replaces a populated
|
|
12858
|
+
directory, so the record is complete or absent and exactly one claimant
|
|
12859
|
+
wins, as with the hard link. The private directory never outlives the
|
|
12860
|
+
attempt."""
|
|
12861
|
+
private = marker.with_name(f".{marker.name}.{claim_id}.d")
|
|
12862
|
+
os.mkdir(private, 0o700)
|
|
12863
|
+
try:
|
|
12864
|
+
fd = os.open(
|
|
12865
|
+
private / _OWNER_RECORD_FILE_NAME,
|
|
12866
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
|
|
12867
|
+
0o600,
|
|
12868
|
+
)
|
|
12869
|
+
try:
|
|
12870
|
+
os.write(fd, payload)
|
|
12871
|
+
os.fsync(fd)
|
|
12872
|
+
finally:
|
|
12873
|
+
os.close(fd)
|
|
12874
|
+
_fsync_directory(private)
|
|
12875
|
+
try:
|
|
12876
|
+
os.rename(private, marker)
|
|
12877
|
+
except OSError as exc:
|
|
12878
|
+
if exc.errno in _OWNER_RECORD_RENAME_CONTENTION:
|
|
12879
|
+
return False
|
|
12880
|
+
raise
|
|
12881
|
+
_fsync_directory(marker.parent)
|
|
12882
|
+
return True
|
|
12883
|
+
finally:
|
|
12884
|
+
try:
|
|
12885
|
+
os.lstat(private)
|
|
12886
|
+
except OSError:
|
|
12887
|
+
pass
|
|
12888
|
+
else:
|
|
12889
|
+
shutil.rmtree(private, ignore_errors=True)
|
|
12890
|
+
|
|
12891
|
+
|
|
12068
12892
|
def _claim_owner_record(
|
|
12069
12893
|
marker: pathlib.Path,
|
|
12070
12894
|
remove_stale: "Callable[[], tuple[bool, str]]",
|
|
@@ -12077,10 +12901,18 @@ def _claim_owner_record(
|
|
|
12077
12901
|
The repair marker's discipline, shared with #901's keeper-yield request:
|
|
12078
12902
|
the record is written and fsynced under a private name and linked into
|
|
12079
12903
|
place, so it is complete or absent; an existing record is replaced only
|
|
12080
|
-
when ``remove_stale`` proves its owner dead or its pid reused.
|
|
12904
|
+
when ``remove_stale`` proves its owner dead or its pid reused. Where the
|
|
12905
|
+
volume has no hard links (#1020) the same record is published as a
|
|
12906
|
+
directory holding it (`_publish_owner_record_directory`); the hard link
|
|
12907
|
+
is tried first on every claim."""
|
|
12081
12908
|
process_start = _process_start_identity(os.getpid())
|
|
12082
12909
|
if process_start is None:
|
|
12083
12910
|
raise OSError(f"could not establish {owner} process identity")
|
|
12911
|
+
# #1020: the removal lock's file exists before the record does, so this
|
|
12912
|
+
# claim's release never needs a new directory entry (a full disk would
|
|
12913
|
+
# otherwise leave this process's own record behind). Creating it is not
|
|
12914
|
+
# taking the lock; publication never takes it.
|
|
12915
|
+
os.close(_open_owner_record_lock_fd(marker))
|
|
12084
12916
|
for _attempt in range(3):
|
|
12085
12917
|
claim_id = f"{os.getpid()}-{time.time_ns()}-{os.urandom(8).hex()}"
|
|
12086
12918
|
temp = marker.with_name(f".{marker.name}.{claim_id}.tmp")
|
|
@@ -12096,14 +12928,32 @@ def _claim_owner_record(
|
|
|
12096
12928
|
finally:
|
|
12097
12929
|
os.close(fd)
|
|
12098
12930
|
try:
|
|
12931
|
+
link_unsupported = False
|
|
12099
12932
|
try:
|
|
12100
|
-
|
|
12933
|
+
_publish_owner_record_link(temp, marker)
|
|
12101
12934
|
except FileExistsError:
|
|
12102
12935
|
removed, reason = remove_stale()
|
|
12103
12936
|
if not removed:
|
|
12104
12937
|
return None, reason
|
|
12105
12938
|
continue
|
|
12106
|
-
|
|
12939
|
+
except OSError as exc:
|
|
12940
|
+
if not _owner_record_link_unsupported(exc):
|
|
12941
|
+
raise
|
|
12942
|
+
link_unsupported = True
|
|
12943
|
+
if link_unsupported:
|
|
12944
|
+
# #1020: no hard links here (exFAT); publish the same
|
|
12945
|
+
# complete record as a directory, by an exclusive rename.
|
|
12946
|
+
if not _publish_owner_record_directory(
|
|
12947
|
+
marker, payload.encode("utf-8"), claim_id,
|
|
12948
|
+
):
|
|
12949
|
+
removed, reason = remove_stale()
|
|
12950
|
+
if not removed:
|
|
12951
|
+
return None, reason
|
|
12952
|
+
continue
|
|
12953
|
+
_trace_owner_record_publication(marker, "directory")
|
|
12954
|
+
else:
|
|
12955
|
+
_fsync_directory(marker.parent)
|
|
12956
|
+
_trace_owner_record_publication(marker, "file")
|
|
12107
12957
|
return RepairMarkerClaim(marker=marker, claim_id=claim_id), ""
|
|
12108
12958
|
finally:
|
|
12109
12959
|
try:
|
|
@@ -12115,14 +12965,15 @@ def _claim_owner_record(
|
|
|
12115
12965
|
|
|
12116
12966
|
def _release_repair_marker(path: pathlib.Path, claim: RepairMarkerClaim) -> None:
|
|
12117
12967
|
marker = _repair_marker_path(path)
|
|
12118
|
-
|
|
12119
|
-
|
|
12120
|
-
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
|
|
12124
|
-
|
|
12125
|
-
|
|
12968
|
+
with _owner_record_removal_lock(marker):
|
|
12969
|
+
# #1020: an absent name was already reclaimed; a newer owner may
|
|
12970
|
+
# publish there at any moment, so it is neither read nor removed.
|
|
12971
|
+
if _owner_record_absent(marker):
|
|
12972
|
+
return
|
|
12973
|
+
owner = _read_repair_owner(marker)
|
|
12974
|
+
if owner is not None and owner.get("claim_id") != claim.claim_id:
|
|
12975
|
+
raise OSError(f"repair marker ownership changed before cleanup: {marker}")
|
|
12976
|
+
_remove_owner_record(marker)
|
|
12126
12977
|
|
|
12127
12978
|
|
|
12128
12979
|
#: #901 W9 (Q15): the one monotonic deadline a process that needs a
|
|
@@ -12141,18 +12992,16 @@ def _remove_stale_keeper_yield_request(
|
|
|
12141
12992
|
"""Remove a provably stale keeper-yield request (dead or reused-pid
|
|
12142
12993
|
owner), as `_remove_stale_repair_marker` does for the repair marker."""
|
|
12143
12994
|
request = _keeper_yield_request_path(path)
|
|
12144
|
-
if not request.exists():
|
|
12145
|
-
return True, ""
|
|
12146
|
-
live, reason = _repair_marker_is_live(request)
|
|
12147
|
-
if live:
|
|
12148
|
-
return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
|
|
12149
12995
|
try:
|
|
12150
|
-
request
|
|
12151
|
-
|
|
12152
|
-
|
|
12996
|
+
with _owner_record_removal_lock(request):
|
|
12997
|
+
if _owner_record_absent(request):
|
|
12998
|
+
return True, ""
|
|
12999
|
+
live, reason = _repair_marker_is_live(request)
|
|
13000
|
+
if live:
|
|
13001
|
+
return False, f"another {path.name} keeper-yield request owns {request} ({reason})"
|
|
13002
|
+
_remove_owner_record(request)
|
|
12153
13003
|
except OSError as exc:
|
|
12154
13004
|
return False, f"could not remove stale keeper-yield request {request}: {exc}"
|
|
12155
|
-
_fsync_directory(request.parent)
|
|
12156
13005
|
return True, reason
|
|
12157
13006
|
|
|
12158
13007
|
|
|
@@ -12175,17 +13024,16 @@ def _release_keeper_yield_request(
|
|
|
12175
13024
|
owns is left alone. Best effort: a request left behind fences keepers
|
|
12176
13025
|
only while this process lives, and it is stale once the process exits."""
|
|
12177
13026
|
request = _keeper_yield_request_path(path)
|
|
12178
|
-
owner = _read_repair_owner(request)
|
|
12179
|
-
if owner is not None and owner.get("claim_id") != claim.claim_id:
|
|
12180
|
-
return
|
|
12181
13027
|
try:
|
|
12182
|
-
request
|
|
13028
|
+
with _owner_record_removal_lock(request):
|
|
13029
|
+
if _owner_record_absent(request):
|
|
13030
|
+
return
|
|
13031
|
+
owner = _read_repair_owner(request)
|
|
13032
|
+
if owner is not None and owner.get("claim_id") != claim.claim_id:
|
|
13033
|
+
return
|
|
13034
|
+
_remove_owner_record(request)
|
|
12183
13035
|
except OSError:
|
|
12184
13036
|
return
|
|
12185
|
-
try:
|
|
12186
|
-
_fsync_directory(request.parent)
|
|
12187
|
-
except OSError:
|
|
12188
|
-
pass
|
|
12189
13037
|
|
|
12190
13038
|
|
|
12191
13039
|
@contextlib.contextmanager
|