cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
package/bin/_cctally_store.py
CHANGED
|
@@ -1784,6 +1784,115 @@ def stats_open_guarded(
|
|
|
1784
1784
|
lock_fh.close()
|
|
1785
1785
|
|
|
1786
1786
|
|
|
1787
|
+
#: #1015 spec 5.3: the fail-fast reader's SQLite busy timeout, in
|
|
1788
|
+
#: milliseconds — a contended read is "unavailable", never a wait.
|
|
1789
|
+
_STATS_FAILFAST_BUSY_TIMEOUT_MS = 50
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
def _stats_failfast_epoch_probe(db_path: pathlib.Path) -> "int | None":
|
|
1793
|
+
"""``PRAGMA user_version`` of the main file, read without locks or the
|
|
1794
|
+
wal-index (see `stats_open_readonly_failfast`); None on any error."""
|
|
1795
|
+
try:
|
|
1796
|
+
probe = sqlite3.connect(
|
|
1797
|
+
db_path.as_uri() + "?mode=ro&immutable=1", uri=True,
|
|
1798
|
+
**_STATS_CONNECT_KWARGS)
|
|
1799
|
+
except sqlite3.Error:
|
|
1800
|
+
return None
|
|
1801
|
+
try:
|
|
1802
|
+
return int(probe.execute("PRAGMA user_version").fetchone()[0])
|
|
1803
|
+
except (sqlite3.Error, TypeError, ValueError):
|
|
1804
|
+
return None
|
|
1805
|
+
finally:
|
|
1806
|
+
probe.close()
|
|
1807
|
+
|
|
1808
|
+
|
|
1809
|
+
def stats_open_readonly_failfast() -> "sqlite3.Connection | None":
|
|
1810
|
+
"""A read-only connection to the PUBLISHED stats index, or None at once.
|
|
1811
|
+
|
|
1812
|
+
#1015 spec 5.3 (review findings R1-P1-4, R2-P1-4, R3-P1-1, R4-P2-3). For a
|
|
1813
|
+
reader that must never change, recover or wait on the stats family — the
|
|
1814
|
+
calibration lineage facts a forecast or the status line consults. It
|
|
1815
|
+
never calls `stats_open_guarded` (which resumes, resolves and recovers),
|
|
1816
|
+
`open_db`, a migration, ingest, a rebuild or a recovery callback.
|
|
1817
|
+
|
|
1818
|
+
1. The maintenance flock is tried SHARED and NON-blocking; busy -> None.
|
|
1819
|
+
2. None, touching nothing, when the repair marker, a pending quarantine
|
|
1820
|
+
record, an interrupted-rebuild artifact, the durable publication
|
|
1821
|
+
marker (pending or failed), or a ``-wal`` / ``-shm`` side file exists.
|
|
1822
|
+
Side files are found by ``stat`` only: no raw descriptor is ever opened
|
|
1823
|
+
on the database file, so this process's SQLite POSIX locks are never
|
|
1824
|
+
released by closing one, and the maintenance drain scanner never sees
|
|
1825
|
+
a descriptor this reader left open.
|
|
1826
|
+
3. ``file:<path>?mode=ro`` as a URI, ``PRAGMA query_only = 1`` and a
|
|
1827
|
+
``busy_timeout`` of `_STATS_FAILFAST_BUSY_TIMEOUT_MS`; None when
|
|
1828
|
+
``PRAGMA user_version`` is not the current stats epoch, or on any
|
|
1829
|
+
SQLite error (a hot rollback journal surfaces as one).
|
|
1830
|
+
4. The shared flock covers the checks and the connect and is released
|
|
1831
|
+
exactly as the guarded opener releases it, so the reader joins the
|
|
1832
|
+
#386 handshake: a destructive maintenance owner drains pre-marker
|
|
1833
|
+
handles.
|
|
1834
|
+
|
|
1835
|
+
R10 (a store last closed by a pre-#538 release keeps a WAL header with no
|
|
1836
|
+
side files) is narrowed by an epoch probe before the real connect: an
|
|
1837
|
+
``immutable=1`` SQLite connection reads ``user_version`` from the main
|
|
1838
|
+
file without the wal-index, so an old-epoch store is refused before a
|
|
1839
|
+
read-only WAL open could create ``-shm`` (32 KiB) and ``-wal``. The probe
|
|
1840
|
+
is a fast reject only; the locked connection re-checks the epoch. It is
|
|
1841
|
+
an SQLite connection, not a raw descriptor: the unix VFS defers closing
|
|
1842
|
+
its descriptor while another connection of this process holds a lock on
|
|
1843
|
+
the same inode, so it releases no lock (the lock-preservation test pins
|
|
1844
|
+
this)."""
|
|
1845
|
+
db_path = pathlib.Path(_cctally_core.DB_PATH)
|
|
1846
|
+
lock_path = pathlib.Path(_cctally_core.STATS_LOCK_MAINTENANCE_PATH)
|
|
1847
|
+
try:
|
|
1848
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1849
|
+
lock_fh = open(lock_path, "a+")
|
|
1850
|
+
except OSError:
|
|
1851
|
+
return None
|
|
1852
|
+
try:
|
|
1853
|
+
try:
|
|
1854
|
+
fcntl.flock(lock_fh, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
1855
|
+
except OSError:
|
|
1856
|
+
return None
|
|
1857
|
+
try:
|
|
1858
|
+
if (_stats_repair_marker(db_path).exists()
|
|
1859
|
+
or _cctally_db._quarantine_pending_path(db_path).exists()
|
|
1860
|
+
or _stats_rebuild_artifact_bases(db_path)
|
|
1861
|
+
or _stats_publication_marker(db_path).exists()
|
|
1862
|
+
or pathlib.Path(str(db_path) + "-wal").exists()
|
|
1863
|
+
or pathlib.Path(str(db_path) + "-shm").exists()
|
|
1864
|
+
or not db_path.exists()):
|
|
1865
|
+
return None
|
|
1866
|
+
conn = None
|
|
1867
|
+
try:
|
|
1868
|
+
epoch = int(_cctally_core.STATS_INDEX_EPOCH)
|
|
1869
|
+
if _stats_failfast_epoch_probe(db_path) != epoch:
|
|
1870
|
+
return None
|
|
1871
|
+
conn = sqlite3.connect(
|
|
1872
|
+
db_path.as_uri() + "?mode=ro", uri=True,
|
|
1873
|
+
timeout=_STATS_FAILFAST_BUSY_TIMEOUT_MS / 1000.0,
|
|
1874
|
+
**_STATS_CONNECT_KWARGS)
|
|
1875
|
+
conn.execute("PRAGMA query_only = 1")
|
|
1876
|
+
# Literal SQL: `_STATS_FAILFAST_BUSY_TIMEOUT_MS` (50).
|
|
1877
|
+
conn.execute("PRAGMA busy_timeout = 50")
|
|
1878
|
+
version = conn.execute("PRAGMA user_version").fetchone()[0]
|
|
1879
|
+
if int(version) != epoch:
|
|
1880
|
+
conn.close()
|
|
1881
|
+
return None
|
|
1882
|
+
return conn
|
|
1883
|
+
except (sqlite3.Error, ValueError):
|
|
1884
|
+
if conn is not None:
|
|
1885
|
+
try:
|
|
1886
|
+
conn.close()
|
|
1887
|
+
except sqlite3.Error:
|
|
1888
|
+
pass
|
|
1889
|
+
return None
|
|
1890
|
+
finally:
|
|
1891
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
1892
|
+
finally:
|
|
1893
|
+
lock_fh.close()
|
|
1894
|
+
|
|
1895
|
+
|
|
1787
1896
|
def _acquire_stats_maintenance_reentrant(path) -> "int | None":
|
|
1788
1897
|
"""Take ``stats.db.maintenance.lock`` EXCLUSIVE unless we already hold it.
|
|
1789
1898
|
|
|
@@ -2092,11 +2201,12 @@ def _stats_heal_hook(
|
|
|
2092
2201
|
)
|
|
2093
2202
|
if reservation == "reserved":
|
|
2094
2203
|
active_request = elected_request or request
|
|
2095
|
-
elif reservation
|
|
2204
|
+
elif reservation in ("pending", "retry-elected"):
|
|
2096
2205
|
# The elected request and coalesced count were read and updated
|
|
2097
2206
|
# while admission was still held. Never reconstruct identity from
|
|
2098
2207
|
# a marker that the worker may not have written yet or may already
|
|
2099
|
-
# have removed.
|
|
2208
|
+
# have removed. `retry-elected` (#1012) keeps that request too: it
|
|
2209
|
+
# only adds a same-request contender.
|
|
2100
2210
|
active_request = elected_request or request
|
|
2101
2211
|
else:
|
|
2102
2212
|
active_request = request
|
|
@@ -2219,6 +2329,11 @@ def _stats_heal_worker_path() -> pathlib.Path:
|
|
|
2219
2329
|
return _stats_heal_path("stats-corruption-heal.worker.lock")
|
|
2220
2330
|
|
|
2221
2331
|
|
|
2332
|
+
def _stats_heal_claim_path() -> pathlib.Path:
|
|
2333
|
+
"""#1012: taken only by heal workers, before the worker lock."""
|
|
2334
|
+
return _stats_heal_path("stats-corruption-heal.claim.lock")
|
|
2335
|
+
|
|
2336
|
+
|
|
2222
2337
|
def _stats_heal_log_path() -> pathlib.Path:
|
|
2223
2338
|
return pathlib.Path(_cctally_core.LOG_DIR) / "stats-corruption-heal.log"
|
|
2224
2339
|
|
|
@@ -2346,27 +2461,15 @@ def _unlink_stats_heal_marker() -> None:
|
|
|
2346
2461
|
|
|
2347
2462
|
|
|
2348
2463
|
def _stats_heal_worker_active() -> bool:
|
|
2349
|
-
"""
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2359
|
-
except BlockingIOError:
|
|
2360
|
-
return True
|
|
2361
|
-
except OSError:
|
|
2362
|
-
return False
|
|
2363
|
-
try:
|
|
2364
|
-
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
2365
|
-
except OSError:
|
|
2366
|
-
pass
|
|
2367
|
-
return False
|
|
2368
|
-
finally:
|
|
2369
|
-
os.close(fd)
|
|
2464
|
+
"""Whether a heal worker may be running, decided from its records.
|
|
2465
|
+
|
|
2466
|
+
#1012: true unless ownership is positively ``absent``, so a live heal
|
|
2467
|
+
whose identity cannot be confirmed keeps the maintenance-timeout message
|
|
2468
|
+
it had under the old lock probe. It takes no lock (neither the worker lock
|
|
2469
|
+
nor the claim lock) and writes nothing: the old try-lock, landing at a
|
|
2470
|
+
worker's claim instant, made that worker exit ``busy``.
|
|
2471
|
+
"""
|
|
2472
|
+
return _rebuild_worker_ownership("heal") != "absent"
|
|
2370
2473
|
|
|
2371
2474
|
|
|
2372
2475
|
def _log_stats_heal(
|
|
@@ -2461,14 +2564,30 @@ def _reserve_stats_corruption_heal(
|
|
|
2461
2564
|
coalescedDetections=elected["coalescedDetections"],
|
|
2462
2565
|
)
|
|
2463
2566
|
return ("pending", elected)
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2567
|
+
ownership = _rebuild_worker_ownership("heal")
|
|
2568
|
+
kept = (_read_stats_heal_request()
|
|
2569
|
+
if ownership == "owned" or (ownership == "uncertain"
|
|
2570
|
+
and age is not None) else None)
|
|
2571
|
+
if ownership == "owned" or kept:
|
|
2572
|
+
# `owned`: a real rebuild outlives the marker retry interval.
|
|
2573
|
+
# Refresh the admission stamp instead of launching a process that
|
|
2574
|
+
# can only lose the claim and exit. `uncertain` (#1012): the heal
|
|
2575
|
+
# may still be live (a null recorded birth, a `ps` failure), and
|
|
2576
|
+
# replacing its request would make that worker's persist and
|
|
2577
|
+
# finalize refuse it. Keep the elected request, coalesce onto it,
|
|
2578
|
+
# and launch a same-request contender (`retry-elected`): the claim
|
|
2579
|
+
# protocol makes it hand off to a live owner or take over from a
|
|
2580
|
+
# dead one. With no marker, or an unreadable one, there is no
|
|
2581
|
+
# request to keep (a corrupt marker carries no healId a live
|
|
2582
|
+
# worker could still persist or finalize against), so an
|
|
2583
|
+
# uncertain record admits a new one below (its worker hands off
|
|
2584
|
+
# the same way); otherwise an unreadable state or marker would
|
|
2585
|
+
# fail every admission and nothing would ever launch the worker
|
|
2586
|
+
# that archives it. `owned` keeps failing on an unreadable
|
|
2587
|
+
# marker: its live worker still holds the request.
|
|
2588
|
+
if not kept:
|
|
2470
2589
|
return ("failed", None)
|
|
2471
|
-
elected = dict(
|
|
2590
|
+
elected = dict(kept)
|
|
2472
2591
|
elected["coalescedDetections"] = int(
|
|
2473
2592
|
elected.get("coalescedDetections") or 0
|
|
2474
2593
|
) + 1
|
|
@@ -2481,7 +2600,8 @@ def _reserve_stats_corruption_heal(
|
|
|
2481
2600
|
wait_s=_STATS_HEAL_ADMISSION_RING_WAIT_S,
|
|
2482
2601
|
coalescedDetections=elected["coalescedDetections"],
|
|
2483
2602
|
)
|
|
2484
|
-
return ("pending"
|
|
2603
|
+
return ("pending" if ownership == "owned" else "retry-elected",
|
|
2604
|
+
elected)
|
|
2485
2605
|
stale_request = _read_stats_heal_request() if age is not None else None
|
|
2486
2606
|
if stale_request:
|
|
2487
2607
|
if not update_stats_heal_event(
|
|
@@ -2541,10 +2661,18 @@ def complete_stats_corruption_heal(reservation: str) -> str:
|
|
|
2541
2661
|
|
|
2542
2662
|
Takes no lock: the spawn need not occur under any, and the maintenance hold
|
|
2543
2663
|
must already have been released so the worker can take it exclusive. A
|
|
2544
|
-
|
|
2545
|
-
admission reports exactly what it
|
|
2664
|
+
reservation other than `reserved` or `retry-elected` is returned
|
|
2665
|
+
unchanged, so a coalesced or failed admission reports exactly what it
|
|
2666
|
+
reported before the split.
|
|
2667
|
+
|
|
2668
|
+
`retry-elected` (#1012) launches a contender for the EXISTING request of
|
|
2669
|
+
an owner whose identity could not be confirmed. It differs from
|
|
2670
|
+
`reserved` only on failure: the marker may belong to a live worker, so a
|
|
2671
|
+
failed spawn is recorded in the ledger but the marker is kept, and the
|
|
2672
|
+
result is `pending` (as is a withheld launch: launch capacity, or a
|
|
2673
|
+
contender already waiting on the same owner).
|
|
2546
2674
|
"""
|
|
2547
|
-
if reservation
|
|
2675
|
+
if reservation not in ("reserved", "retry-elected"):
|
|
2548
2676
|
return reservation
|
|
2549
2677
|
# #997 D8 launch: the reservation is persisted under the admission flock
|
|
2550
2678
|
# before the spawn, and the child's identity filled under it after. The
|
|
@@ -2557,14 +2685,22 @@ def complete_stats_corruption_heal(reservation: str) -> str:
|
|
|
2557
2685
|
_with_heal_admission(lambda: reserve.append(_spawner_reserve(
|
|
2558
2686
|
"heal", launch_id, _cctally_core.now_utc_iso())))
|
|
2559
2687
|
if reserve == ["withheld"]:
|
|
2560
|
-
# Launch capacity
|
|
2561
|
-
#
|
|
2688
|
+
# Launch capacity, or (#1012) a contender that has not started
|
|
2689
|
+
# already waits on an owner not proven gone: the admitted request
|
|
2690
|
+
# stays pending (its marker is kept) and nothing is spawned; a later
|
|
2691
|
+
# detection retries it.
|
|
2562
2692
|
return "pending"
|
|
2563
2693
|
pid = _spawn_rebuild_worker(STATS_CORRUPTION_HEAL_COMMAND, launch_id)
|
|
2564
2694
|
if pid is not None:
|
|
2565
2695
|
_with_heal_admission(lambda: _spawner_fill(
|
|
2566
2696
|
"heal", launch_id, pid, _cctally_core.now_utc_iso()))
|
|
2567
2697
|
return "spawned"
|
|
2698
|
+
if reservation == "retry-elected":
|
|
2699
|
+
# The elected request may still belong to a live worker: record the
|
|
2700
|
+
# failed start, keep its marker, and leave the request pending.
|
|
2701
|
+
_with_heal_admission(lambda: _spawner_fill(
|
|
2702
|
+
"heal", launch_id, None, _cctally_core.now_utc_iso()))
|
|
2703
|
+
return "pending"
|
|
2568
2704
|
# Drop our own marker so the next detection is admitted immediately rather
|
|
2569
2705
|
# than waiting out the retry window. Done under the admission flock, as it
|
|
2570
2706
|
# was before the split, so no concurrent caller can coalesce onto a marker
|
|
@@ -2869,20 +3005,12 @@ def cmd_stats_corruption_heal_internal(args) -> int:
|
|
|
2869
3005
|
"""Hidden detached worker: heal one corrupt stats index exactly once."""
|
|
2870
3006
|
del args
|
|
2871
3007
|
healed = False
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
)
|
|
2877
|
-
except OSError as exc:
|
|
2878
|
-
_log_stats_heal("error", error=exc)
|
|
3008
|
+
# #1012: the claim lock, then the worker lock; a loser records `busy`
|
|
3009
|
+
# only against a live started owner (spec 5.3).
|
|
3010
|
+
claim = _claim_rebuild_worker("heal")
|
|
3011
|
+
if isinstance(claim, str):
|
|
2879
3012
|
return 0
|
|
2880
3013
|
try:
|
|
2881
|
-
try:
|
|
2882
|
-
fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2883
|
-
except OSError:
|
|
2884
|
-
_record_worker_busy("heal")
|
|
2885
|
-
return 0
|
|
2886
3014
|
# #997 D8: record the run (and any dead predecessor) before any work;
|
|
2887
3015
|
# a run that cannot be recorded does none.
|
|
2888
3016
|
run = _WorkerRun("heal")
|
|
@@ -2898,14 +3026,10 @@ def cmd_stats_corruption_heal_internal(args) -> int:
|
|
|
2898
3026
|
finalization=result["finalization"],
|
|
2899
3027
|
decline_reason=result["declineReason"])
|
|
2900
3028
|
finally:
|
|
2901
|
-
|
|
2902
|
-
fcntl.flock(worker_fd, fcntl.LOCK_UN)
|
|
2903
|
-
except OSError:
|
|
2904
|
-
pass
|
|
2905
|
-
os.close(worker_fd)
|
|
3029
|
+
_release_rebuild_worker_claim(claim)
|
|
2906
3030
|
if healed:
|
|
2907
3031
|
# #825/#832 spec 4.3 revision 2: the healed index posts its
|
|
2908
|
-
# completion wake, with the worker
|
|
3032
|
+
# completion wake, with the worker and claim locks released.
|
|
2909
3033
|
import _cctally_completion
|
|
2910
3034
|
_cctally_completion.post_publication_wake()
|
|
2911
3035
|
|
|
@@ -3197,6 +3321,11 @@ def _stats_epoch_rebuild_worker_path() -> pathlib.Path:
|
|
|
3197
3321
|
return _stats_epoch_rebuild_path("stats-epoch-rebuild.worker.lock")
|
|
3198
3322
|
|
|
3199
3323
|
|
|
3324
|
+
def _stats_epoch_rebuild_claim_path() -> pathlib.Path:
|
|
3325
|
+
"""#1012: taken only by epoch workers, before the worker lock."""
|
|
3326
|
+
return _stats_epoch_rebuild_path("stats-epoch-rebuild.claim.lock")
|
|
3327
|
+
|
|
3328
|
+
|
|
3200
3329
|
def _stats_epoch_rebuild_log_path() -> pathlib.Path:
|
|
3201
3330
|
return pathlib.Path(_cctally_core.LOG_DIR) / "stats-epoch-rebuild.log"
|
|
3202
3331
|
|
|
@@ -3209,29 +3338,16 @@ def _unlink_stats_epoch_marker() -> None:
|
|
|
3209
3338
|
|
|
3210
3339
|
|
|
3211
3340
|
def _stats_epoch_rebuild_worker_active() -> bool:
|
|
3212
|
-
"""
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
try:
|
|
3223
|
-
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
3224
|
-
except BlockingIOError:
|
|
3225
|
-
return True
|
|
3226
|
-
except OSError:
|
|
3227
|
-
return False
|
|
3228
|
-
try:
|
|
3229
|
-
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
3230
|
-
except OSError:
|
|
3231
|
-
pass
|
|
3232
|
-
return False
|
|
3233
|
-
finally:
|
|
3234
|
-
os.close(fd)
|
|
3341
|
+
"""Whether a positively identified epoch worker owns the rebuild.
|
|
3342
|
+
|
|
3343
|
+
#1012: decided from the run state, launch ledger and process identity
|
|
3344
|
+
(``_rebuild_worker_ownership == "owned"``); an uncertain owner reads as
|
|
3345
|
+
inactive, so its caller launches a contender that the claim protocol makes
|
|
3346
|
+
hand off to a live owner. It takes no lock and writes nothing: the old
|
|
3347
|
+
try-lock of the worker flock, landing at a worker's claim instant, made
|
|
3348
|
+
that worker exit ``busy`` and left the pending rebuild unowned.
|
|
3349
|
+
"""
|
|
3350
|
+
return _rebuild_worker_ownership("epoch") == "owned"
|
|
3235
3351
|
|
|
3236
3352
|
|
|
3237
3353
|
def _log_stats_epoch_rebuild(outcome: str, *, error: Exception | None = None) -> None:
|
|
@@ -3315,6 +3431,18 @@ def _worker_admission_path(worker: str) -> pathlib.Path:
|
|
|
3315
3431
|
return _stats_heal_admission_path()
|
|
3316
3432
|
|
|
3317
3433
|
|
|
3434
|
+
def _worker_claim_path(worker: str) -> pathlib.Path:
|
|
3435
|
+
if worker == "epoch":
|
|
3436
|
+
return _stats_epoch_rebuild_claim_path()
|
|
3437
|
+
return _stats_heal_claim_path()
|
|
3438
|
+
|
|
3439
|
+
|
|
3440
|
+
def _worker_lock_path(worker: str) -> pathlib.Path:
|
|
3441
|
+
if worker == "epoch":
|
|
3442
|
+
return _stats_epoch_rebuild_worker_path()
|
|
3443
|
+
return _stats_heal_worker_path()
|
|
3444
|
+
|
|
3445
|
+
|
|
3318
3446
|
def _read_private_record(path: pathlib.Path):
|
|
3319
3447
|
"""``(payload | None, unreadable, raw bytes | None)`` for one record.
|
|
3320
3448
|
|
|
@@ -3440,12 +3568,13 @@ def read_stats_heal_request_snapshot() -> "dict | None":
|
|
|
3440
3568
|
}
|
|
3441
3569
|
|
|
3442
3570
|
|
|
3443
|
-
def process_identity(pid) -> dict:
|
|
3571
|
+
def process_identity(pid, *, timeout: float = 10) -> dict:
|
|
3444
3572
|
"""``{"status": "alive"|"gone"|"unknown", "birth": str | None}``.
|
|
3445
3573
|
|
|
3446
3574
|
From ``ps -o stat=,lstart= -p <pid>`` under ``LC_ALL=C``: no such process
|
|
3447
3575
|
(``ps`` exits 1 with no output) or a zombie (``Z`` in ``stat``) is
|
|
3448
|
-
``gone``; ``ps`` failing in any other way
|
|
3576
|
+
``gone``; ``ps`` failing in any other way, including running past
|
|
3577
|
+
``timeout`` seconds, is ``unknown``. ``birth`` is the
|
|
3449
3578
|
one-second-resolution start time, whitespace-normalized.
|
|
3450
3579
|
"""
|
|
3451
3580
|
import subprocess
|
|
@@ -3455,7 +3584,7 @@ def process_identity(pid) -> dict:
|
|
|
3455
3584
|
try:
|
|
3456
3585
|
out = subprocess.run(
|
|
3457
3586
|
["ps", "-o", "stat=,lstart=", "-p", str(pid)],
|
|
3458
|
-
capture_output=True, text=True, timeout=
|
|
3587
|
+
capture_output=True, text=True, timeout=timeout, check=False,
|
|
3459
3588
|
env={**os.environ, "LC_ALL": "C"})
|
|
3460
3589
|
except (OSError, subprocess.SubprocessError):
|
|
3461
3590
|
return {"status": "unknown", "birth": None}
|
|
@@ -3471,6 +3600,91 @@ def process_identity(pid) -> dict:
|
|
|
3471
3600
|
return {"status": "alive", "birth": birth}
|
|
3472
3601
|
|
|
3473
3602
|
|
|
3603
|
+
#: #1012: the ownership check's own bound on its ``ps`` call, well below the
|
|
3604
|
+
#: diagnostics' 10 s. An identity that times out is ``unknown``, so the answer
|
|
3605
|
+
#: is ``uncertain``; an admission never stalls on ``ps``.
|
|
3606
|
+
_OWNERSHIP_IDENTITY_TIMEOUT_S = 2.0
|
|
3607
|
+
|
|
3608
|
+
#: Run fields a live worker rewrites as it progresses. A change to these alone
|
|
3609
|
+
#: between an ownership check's two reads is not a change of owner.
|
|
3610
|
+
_RUN_PROGRESS_FIELDS = frozenset({
|
|
3611
|
+
"phase", "phaseStartedAt", "progressAt", "counters", "residentBytes",
|
|
3612
|
+
"peakResidentBytes",
|
|
3613
|
+
})
|
|
3614
|
+
|
|
3615
|
+
|
|
3616
|
+
def _ownership_records(worker: str):
|
|
3617
|
+
"""One read of the run state, then the ledger, for an ownership check.
|
|
3618
|
+
|
|
3619
|
+
``(verdict, run, entry)``: ``verdict`` is ``"absent"`` or ``"uncertain"``
|
|
3620
|
+
when the records alone settle the answer, else ``None`` for a well-formed
|
|
3621
|
+
running run whose unresolved ledger entry acknowledges ``started`` with
|
|
3622
|
+
its ``runId``. Never writes.
|
|
3623
|
+
"""
|
|
3624
|
+
state = read_worker_state(worker)
|
|
3625
|
+
if state["unreadable"]:
|
|
3626
|
+
return "uncertain", None, None
|
|
3627
|
+
run = (state["state"] or {}).get("run")
|
|
3628
|
+
if run is None or run.get("status") == "finished":
|
|
3629
|
+
return "absent", None, None
|
|
3630
|
+
pid = run.get("pid")
|
|
3631
|
+
if (run.get("status") != "running"
|
|
3632
|
+
or not isinstance(run.get("runId"), str)
|
|
3633
|
+
or not isinstance(run.get("launchId"), str)
|
|
3634
|
+
or not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0):
|
|
3635
|
+
return "uncertain", run, None
|
|
3636
|
+
ledger = read_launch_ledger(worker)
|
|
3637
|
+
if ledger["unreadable"]:
|
|
3638
|
+
return "uncertain", run, None
|
|
3639
|
+
entry = _wl.find(ledger["ledger"] or {}, run["launchId"])
|
|
3640
|
+
if entry is not None and entry.get("outcome"):
|
|
3641
|
+
# A recorded terminal is authoritative, as in `_reconcile_prior_run`.
|
|
3642
|
+
return "absent", run, entry
|
|
3643
|
+
ack = (entry or {}).get("ack")
|
|
3644
|
+
if (not isinstance(ack, dict) or ack.get("kind") != "started"
|
|
3645
|
+
or ack.get("runId") != run["runId"]):
|
|
3646
|
+
return "uncertain", run, entry
|
|
3647
|
+
return None, run, entry
|
|
3648
|
+
|
|
3649
|
+
|
|
3650
|
+
def _rebuild_worker_ownership(worker: str) -> str:
|
|
3651
|
+
"""Who owns the worker's slot, from records alone (#1012).
|
|
3652
|
+
|
|
3653
|
+
``"owned"`` only on positive evidence: a running run, its unresolved
|
|
3654
|
+
ledger entry acknowledging ``started`` with that ``runId``, and the run's
|
|
3655
|
+
pid alive with the recorded birth, confirmed by a second read of both
|
|
3656
|
+
records after the identity probe (progress-only updates are ignored).
|
|
3657
|
+
``"absent"`` when no run is running, its entry is terminal, or its process
|
|
3658
|
+
is confirmed gone. ``"uncertain"`` otherwise: unreadable or malformed
|
|
3659
|
+
records, an entry that is missing or acknowledges another run, a null
|
|
3660
|
+
recorded birth, a ``ps`` failure or timeout, or a run that changed under
|
|
3661
|
+
the probe without finishing.
|
|
3662
|
+
|
|
3663
|
+
It never opens a lock file and never writes, so no check can make a
|
|
3664
|
+
starting worker lose its claim. Callers map the three answers (spec 5.2):
|
|
3665
|
+
an uncertain owner may never authorize a ``busy`` exit, and the heal
|
|
3666
|
+
admission never replaces an uncertain owner's request.
|
|
3667
|
+
"""
|
|
3668
|
+
verdict, run, entry = _ownership_records(worker)
|
|
3669
|
+
if verdict is not None:
|
|
3670
|
+
return verdict
|
|
3671
|
+
pid = run["pid"]
|
|
3672
|
+
identity = process_identity(pid, timeout=_OWNERSHIP_IDENTITY_TIMEOUT_S)
|
|
3673
|
+
liveness = _wl.liveness(run, {pid: identity})
|
|
3674
|
+
if liveness == "gone":
|
|
3675
|
+
return "absent"
|
|
3676
|
+
if liveness != "alive":
|
|
3677
|
+
return "uncertain"
|
|
3678
|
+
again, run_again, entry_again = _ownership_records(worker)
|
|
3679
|
+
if again is not None:
|
|
3680
|
+
return again
|
|
3681
|
+
unchanged = (
|
|
3682
|
+
{k: v for k, v in run_again.items() if k not in _RUN_PROGRESS_FIELDS}
|
|
3683
|
+
== {k: v for k, v in run.items() if k not in _RUN_PROGRESS_FIELDS}
|
|
3684
|
+
and entry_again.get("ack") == entry.get("ack"))
|
|
3685
|
+
return "owned" if unchanged else "uncertain"
|
|
3686
|
+
|
|
3687
|
+
|
|
3474
3688
|
def _archive_unreadable_record(path: pathlib.Path, raw) -> "str | None":
|
|
3475
3689
|
"""Copy an unreadable record's bytes to ``<name>.unreadable-<UTC stamp>``
|
|
3476
3690
|
(a new private file, ``fsync``ed); the original stays in place until its
|
|
@@ -3541,23 +3755,56 @@ def _spawner_classify(worker: str, ledger: dict, now: str) -> dict:
|
|
|
3541
3755
|
return _wl.classify(ledger, state, identities, now=now)["ledger"]
|
|
3542
3756
|
|
|
3543
3757
|
|
|
3758
|
+
def _contender_on_standby(ledger: dict, state, now: str) -> bool:
|
|
3759
|
+
"""#1012: whether a contender already waits on an unsettled owner.
|
|
3760
|
+
|
|
3761
|
+
True when the run state's run is ``running`` and its launch is not
|
|
3762
|
+
settled in the classified ``ledger`` (an absent entry, or one with no
|
|
3763
|
+
outcome: the owner is not proven gone or finished), AND another
|
|
3764
|
+
unresolved launch has no acknowledgement (it has not started) and was
|
|
3765
|
+
reserved within ``_CONTENDER_STANDBY_S`` before ``now``. An
|
|
3766
|
+
unparseable reservation time never withholds."""
|
|
3767
|
+
run = state.get("run") if isinstance(state, dict) else None
|
|
3768
|
+
if not isinstance(run, dict) or run.get("status") != "running":
|
|
3769
|
+
return False
|
|
3770
|
+
own = run.get("launchId")
|
|
3771
|
+
entry = _wl.find(ledger, own) if isinstance(own, str) else None
|
|
3772
|
+
if entry is not None and entry.get("outcome"):
|
|
3773
|
+
return False
|
|
3774
|
+
now_at = _parse_utc_instant(now)
|
|
3775
|
+
if now_at is None:
|
|
3776
|
+
return False
|
|
3777
|
+
for other in _wl.unresolved(ledger):
|
|
3778
|
+
if other.get("launchId") == own or other.get("ack") is not None:
|
|
3779
|
+
continue
|
|
3780
|
+
at = _parse_utc_instant(other.get("at"))
|
|
3781
|
+
if at is not None and (0 <= (now_at - at).total_seconds()
|
|
3782
|
+
< _CONTENDER_STANDBY_S):
|
|
3783
|
+
return True
|
|
3784
|
+
return False
|
|
3785
|
+
|
|
3786
|
+
|
|
3544
3787
|
def _spawner_reserve(worker: str, launch_id: str, now: str) -> str:
|
|
3545
3788
|
"""Classify the ledger and persist this launch's reservation (caller
|
|
3546
3789
|
holds the admission flock).
|
|
3547
3790
|
|
|
3548
|
-
``"reserved"`` when the reservation is durable; ``"withheld"`` when
|
|
3791
|
+
``"reserved"`` when the reservation is durable; ``"withheld"`` when the
|
|
3792
|
+
caller must not spawn (the settlements are written, no reservation is,
|
|
3793
|
+
and the request stays pending), for either of two reasons:
|
|
3549
3794
|
``LAUNCH_CAPACITY`` entries are still unresolved after classification
|
|
3550
|
-
(
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
the
|
|
3795
|
+
(`doctor` reports ``details.admission = "launch-capacity"``), or (#1012)
|
|
3796
|
+
a contender that has not started already waits on a recorded owner that
|
|
3797
|
+
is not proven gone or finished (``_contender_on_standby``), so a further
|
|
3798
|
+
one would only wait beside it; ``"unwritten"`` when the ledger could not
|
|
3799
|
+
be written, in which case the caller still spawns and the worker's
|
|
3800
|
+
acknowledgement records the launch."""
|
|
3555
3801
|
try:
|
|
3556
3802
|
ledger = _load_ledger_for_update(worker)
|
|
3557
3803
|
ledger = _spawner_classify(worker, ledger, now)
|
|
3558
3804
|
state = read_worker_state(worker)["state"]
|
|
3559
3805
|
protected = _current_run_launch_id(state)
|
|
3560
|
-
if not _wl.has_launch_capacity(ledger)
|
|
3806
|
+
if (not _wl.has_launch_capacity(ledger)
|
|
3807
|
+
or _contender_on_standby(ledger, state, now)):
|
|
3561
3808
|
_write_ledger(worker, _wl.prune(ledger, protected))
|
|
3562
3809
|
return "withheld"
|
|
3563
3810
|
ledger = _wl.reserve(ledger, launch_id=launch_id, at=now)
|
|
@@ -3672,6 +3919,194 @@ def _record_worker_busy(worker: str) -> None:
|
|
|
3672
3919
|
_heal_release_flock(fd)
|
|
3673
3920
|
|
|
3674
3921
|
|
|
3922
|
+
# --------------------------------------------------------------------------
|
|
3923
|
+
# #1012 — the claim protocol (spec 5.3)
|
|
3924
|
+
# --------------------------------------------------------------------------
|
|
3925
|
+
#
|
|
3926
|
+
# Only the two worker entry points take a claim lock: no probe, observer or
|
|
3927
|
+
# diagnostic may, or a check would again be able to defeat a starting worker.
|
|
3928
|
+
# A worker holds its claim lock for as long as it holds its worker lock, so a
|
|
3929
|
+
# claimant that finds the worker lock busy is looking at an observer (waited
|
|
3930
|
+
# out) or an older binary's worker (identified by its records: `busy`). The
|
|
3931
|
+
# one other owner its records can identify is a current worker after a double
|
|
3932
|
+
# fault: its `finish()` failed both the state write and the ledger settle,
|
|
3933
|
+
# and its process still lives.
|
|
3934
|
+
# Lock order: claim lock -> worker lock -> the admission / maintenance /
|
|
3935
|
+
# ingest order of docs/journal-gotchas.md.
|
|
3936
|
+
|
|
3937
|
+
#: A claimant retries a contended worker lock this often, with no elapsed-time
|
|
3938
|
+
#: bound: a non-owner hold delays the claimant and never defeats it.
|
|
3939
|
+
_CLAIM_RETRY_S = 0.02
|
|
3940
|
+
#: An admission launches no further contender while one that has not started
|
|
3941
|
+
#: (no acknowledgement) was reserved this recently for an owner not proven
|
|
3942
|
+
#: gone or finished (``_contender_on_standby``). That waiting contender's
|
|
3943
|
+
#: body starts after the current obligation and serves it, so a second would
|
|
3944
|
+
#: only wait beside it; a dead contender of unknown liveness delays a new
|
|
3945
|
+
#: launch by at most this long.
|
|
3946
|
+
_CONTENDER_STANDBY_S = 1800.0
|
|
3947
|
+
#: ...re-evaluating that lock's ownership at most this often (and at once on
|
|
3948
|
+
#: its first refusal). A claim loser's ownership check starts at this period.
|
|
3949
|
+
_CLAIM_OWNER_RECHECK_S = 1.0
|
|
3950
|
+
#: A claim loser sleeps a jittered first interval in this range, doubling to
|
|
3951
|
+
#: the cap; its ownership check backs off to once per the recheck cap.
|
|
3952
|
+
_HANDOFF_FIRST_SLEEP_S = (0.01, 0.03)
|
|
3953
|
+
_HANDOFF_SLEEP_CAP_S = 1.0
|
|
3954
|
+
_HANDOFF_RECHECK_CAP_S = 30.0
|
|
3955
|
+
|
|
3956
|
+
|
|
3957
|
+
def _unlock_close(fd: int) -> None:
|
|
3958
|
+
try:
|
|
3959
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
3960
|
+
except OSError:
|
|
3961
|
+
pass
|
|
3962
|
+
finally:
|
|
3963
|
+
os.close(fd)
|
|
3964
|
+
|
|
3965
|
+
|
|
3966
|
+
def _hand_off_rebuild_worker_claim(worker: str, claim_fd: int) -> "str | None":
|
|
3967
|
+
"""A claim loser's wait (spec 5.3 step 1): ``None`` once ``claim_fd``
|
|
3968
|
+
holds the claim; ``"busy"`` (after ``_record_worker_busy``) once a live
|
|
3969
|
+
worker that has started holds the worker lock; ``"error"`` on an I/O
|
|
3970
|
+
error that is not contention.
|
|
3971
|
+
|
|
3972
|
+
Only positive ownership authorizes ``busy``: a holder between its two
|
|
3973
|
+
flocks or before ``begin()``, a stale record and an unconfirmable
|
|
3974
|
+
identity all keep the loser waiting, so the pending work never loses its
|
|
3975
|
+
last claimant. Its worker-lock try is released at once, it holds no lock
|
|
3976
|
+
while it sleeps, and nothing bounds the wait but the holder's release.
|
|
3977
|
+
"""
|
|
3978
|
+
import random
|
|
3979
|
+
|
|
3980
|
+
sleep_s = random.uniform(*_HANDOFF_FIRST_SLEEP_S)
|
|
3981
|
+
recheck_s = _CLAIM_OWNER_RECHECK_S
|
|
3982
|
+
next_check = None
|
|
3983
|
+
while True:
|
|
3984
|
+
try:
|
|
3985
|
+
worker_fd = os.open(_worker_lock_path(worker),
|
|
3986
|
+
os.O_WRONLY | os.O_CREAT, 0o600)
|
|
3987
|
+
except OSError as exc:
|
|
3988
|
+
_worker_log_error(worker, "claim", exc)
|
|
3989
|
+
return "error"
|
|
3990
|
+
try:
|
|
3991
|
+
try:
|
|
3992
|
+
fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
3993
|
+
except BlockingIOError:
|
|
3994
|
+
contended = True
|
|
3995
|
+
except OSError as exc:
|
|
3996
|
+
_worker_log_error(worker, "claim", exc)
|
|
3997
|
+
return "error"
|
|
3998
|
+
else:
|
|
3999
|
+
contended = False
|
|
4000
|
+
# Closing the descriptor releases the try anyway; an unlock
|
|
4001
|
+
# error must not escape the worker.
|
|
4002
|
+
try:
|
|
4003
|
+
fcntl.flock(worker_fd, fcntl.LOCK_UN)
|
|
4004
|
+
except OSError:
|
|
4005
|
+
pass
|
|
4006
|
+
finally:
|
|
4007
|
+
os.close(worker_fd)
|
|
4008
|
+
if contended:
|
|
4009
|
+
now = time.monotonic()
|
|
4010
|
+
if next_check is None or now >= next_check:
|
|
4011
|
+
if _rebuild_worker_ownership(worker) == "owned":
|
|
4012
|
+
_record_worker_busy(worker)
|
|
4013
|
+
return "busy"
|
|
4014
|
+
next_check = now + recheck_s
|
|
4015
|
+
recheck_s = min(recheck_s * 2, _HANDOFF_RECHECK_CAP_S)
|
|
4016
|
+
time.sleep(sleep_s)
|
|
4017
|
+
sleep_s = min(sleep_s * 2, _HANDOFF_SLEEP_CAP_S)
|
|
4018
|
+
try:
|
|
4019
|
+
fcntl.flock(claim_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4020
|
+
except BlockingIOError:
|
|
4021
|
+
continue
|
|
4022
|
+
except OSError as exc:
|
|
4023
|
+
_worker_log_error(worker, "claim", exc)
|
|
4024
|
+
return "error"
|
|
4025
|
+
return None
|
|
4026
|
+
|
|
4027
|
+
|
|
4028
|
+
def _claim_rebuild_worker(worker: str) -> "tuple[int, int] | str":
|
|
4029
|
+
"""Take the worker's claim lock, then its worker lock (spec 5.3).
|
|
4030
|
+
|
|
4031
|
+
Returns the held ``(claim_fd, worker_fd)`` for
|
|
4032
|
+
``_release_rebuild_worker_claim``, or ``"busy"`` (recorded through
|
|
4033
|
+
``_record_worker_busy``) or ``"error"`` (logged ``stage=claim``, with no
|
|
4034
|
+
acknowledgement). A claim loser hands off before any ``busy``; a claimant
|
|
4035
|
+
waits out any hold of the worker lock except a positively identified
|
|
4036
|
+
owner. While this process holds the claim that owner is an older
|
|
4037
|
+
binary's worker, or, after a double fault, a current one whose
|
|
4038
|
+
``finish()`` failed both its state write and its ledger settle while its
|
|
4039
|
+
process lives on: its stale ``running`` record then names a live pid.
|
|
4040
|
+
"""
|
|
4041
|
+
fds: list = [] # open descriptors not handed to the caller, claim first
|
|
4042
|
+
|
|
4043
|
+
def release_all() -> None:
|
|
4044
|
+
while fds:
|
|
4045
|
+
_unlock_close(fds.pop()) # the worker lock first, then the claim
|
|
4046
|
+
|
|
4047
|
+
def failed(exc) -> str:
|
|
4048
|
+
release_all()
|
|
4049
|
+
_worker_log_error(worker, "claim", exc)
|
|
4050
|
+
return "error"
|
|
4051
|
+
|
|
4052
|
+
try:
|
|
4053
|
+
try:
|
|
4054
|
+
pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True,
|
|
4055
|
+
exist_ok=True)
|
|
4056
|
+
fds.append(os.open(_worker_claim_path(worker),
|
|
4057
|
+
os.O_WRONLY | os.O_CREAT, 0o600))
|
|
4058
|
+
except OSError as exc:
|
|
4059
|
+
return failed(exc)
|
|
4060
|
+
claim_fd = fds[0]
|
|
4061
|
+
try:
|
|
4062
|
+
fcntl.flock(claim_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4063
|
+
except BlockingIOError:
|
|
4064
|
+
handed = _hand_off_rebuild_worker_claim(worker, claim_fd)
|
|
4065
|
+
if handed is not None:
|
|
4066
|
+
return handed
|
|
4067
|
+
except OSError as exc:
|
|
4068
|
+
return failed(exc)
|
|
4069
|
+
try:
|
|
4070
|
+
fds.append(os.open(_worker_lock_path(worker),
|
|
4071
|
+
os.O_WRONLY | os.O_CREAT, 0o600))
|
|
4072
|
+
except OSError as exc:
|
|
4073
|
+
return failed(exc)
|
|
4074
|
+
worker_fd = fds[1]
|
|
4075
|
+
next_check = None
|
|
4076
|
+
while True:
|
|
4077
|
+
try:
|
|
4078
|
+
fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4079
|
+
except BlockingIOError:
|
|
4080
|
+
pass
|
|
4081
|
+
except OSError as exc:
|
|
4082
|
+
return failed(exc)
|
|
4083
|
+
else:
|
|
4084
|
+
fds.clear()
|
|
4085
|
+
return (claim_fd, worker_fd)
|
|
4086
|
+
now = time.monotonic()
|
|
4087
|
+
if next_check is None or now >= next_check:
|
|
4088
|
+
next_check = now + _CLAIM_OWNER_RECHECK_S
|
|
4089
|
+
if _rebuild_worker_ownership(worker) == "owned":
|
|
4090
|
+
release_all()
|
|
4091
|
+
_record_worker_busy(worker)
|
|
4092
|
+
return "busy"
|
|
4093
|
+
time.sleep(_CLAIM_RETRY_S)
|
|
4094
|
+
finally:
|
|
4095
|
+
release_all()
|
|
4096
|
+
|
|
4097
|
+
|
|
4098
|
+
def _release_rebuild_worker_claim(claim: "tuple[int, int]") -> None:
|
|
4099
|
+
"""Release a held claim: the worker lock first, then the claim lock, and
|
|
4100
|
+
both before any completion wake. Releasing the claim first would let a
|
|
4101
|
+
second claimant hold it while this process still held the worker lock,
|
|
4102
|
+
and misread this process as an older binary's owner."""
|
|
4103
|
+
claim_fd, worker_fd = claim
|
|
4104
|
+
try:
|
|
4105
|
+
_unlock_close(worker_fd)
|
|
4106
|
+
finally:
|
|
4107
|
+
_unlock_close(claim_fd)
|
|
4108
|
+
|
|
4109
|
+
|
|
3675
4110
|
def rebuild_worker_snapshot(worker: str) -> dict:
|
|
3676
4111
|
"""What `doctor` and `db status` classify, gathered without any lock
|
|
3677
4112
|
and without writing (spec D8 "Liveness without the flock").
|
|
@@ -4060,8 +4495,10 @@ def defer_stats_epoch_rebuild() -> str:
|
|
|
4060
4495
|
# launch id, then fill the child's identity (or record the failure).
|
|
4061
4496
|
launch_id = _new_launch_id()
|
|
4062
4497
|
if _spawner_reserve("epoch", launch_id, now) == "withheld":
|
|
4063
|
-
# Launch capacity
|
|
4064
|
-
#
|
|
4498
|
+
# Launch capacity, or (#1012) a contender that has not started
|
|
4499
|
+
# already waits on an owner not proven gone: the marker (just
|
|
4500
|
+
# refreshed, so the retry throttle holds) keeps the request
|
|
4501
|
+
# pending; nothing is spawned.
|
|
4065
4502
|
return "pending"
|
|
4066
4503
|
pid = _spawn_rebuild_worker(STATS_EPOCH_REBUILD_COMMAND, launch_id)
|
|
4067
4504
|
_spawner_fill("epoch", launch_id, pid, _cctally_core.now_utc_iso())
|
|
@@ -4083,7 +4520,18 @@ def stats_rebuild_worker_blocks() -> tuple:
|
|
|
4083
4520
|
`_cctally_journal.LockBlock` naming the worker lock that refused it,
|
|
4084
4521
|
whether it was held (contention) or the probe hit an I/O error. A worker
|
|
4085
4522
|
lock that is free, or whose file does not exist, yields nothing: an
|
|
4086
|
-
unlocked probe proves no release.
|
|
4523
|
+
unlocked probe proves no release.
|
|
4524
|
+
|
|
4525
|
+
#1012: this stays lock-based, unlike the ownership probes, which never
|
|
4526
|
+
acquire a worker lock. Actual contention (including the window before a
|
|
4527
|
+
worker's run record exists) is what chooses between a release obligation
|
|
4528
|
+
and an inline rebuild, and a stale record must not invent an obligation.
|
|
4529
|
+
This function and the statusline dependency observers take only the
|
|
4530
|
+
worker (compatibility) lock, never a claim lock; the claim protocol keeps
|
|
4531
|
+
their brief holds from defeating a claimant, and a claim loser returns
|
|
4532
|
+
`busy` only while a positively identified live worker holds the worker
|
|
4533
|
+
lock this function observes (a holder may still release between that
|
|
4534
|
+
`busy` and this probe: spec residual R5)."""
|
|
4087
4535
|
import _cctally_journal
|
|
4088
4536
|
|
|
4089
4537
|
blocks = []
|
|
@@ -4134,22 +4582,12 @@ STATS_EPOCH_RESIDUAL_NOT_DURABLE = "success-residual-not-durable"
|
|
|
4134
4582
|
def _stats_epoch_rebuild_worker(args) -> str:
|
|
4135
4583
|
"""The worker body; returns its outcome."""
|
|
4136
4584
|
del args
|
|
4585
|
+
# #1012: the claim lock, then the worker lock; `busy` only against a live
|
|
4586
|
+
# started owner, `error` for a lock I/O failure (spec 5.3).
|
|
4587
|
+
claim = _claim_rebuild_worker("epoch")
|
|
4588
|
+
if isinstance(claim, str):
|
|
4589
|
+
return claim
|
|
4137
4590
|
try:
|
|
4138
|
-
pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True, exist_ok=True)
|
|
4139
|
-
worker_fd = os.open(
|
|
4140
|
-
_stats_epoch_rebuild_worker_path(),
|
|
4141
|
-
os.O_WRONLY | os.O_CREAT,
|
|
4142
|
-
0o600,
|
|
4143
|
-
)
|
|
4144
|
-
except OSError as exc:
|
|
4145
|
-
_log_stats_epoch_rebuild("error", error=exc)
|
|
4146
|
-
return "error"
|
|
4147
|
-
try:
|
|
4148
|
-
try:
|
|
4149
|
-
fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
4150
|
-
except OSError:
|
|
4151
|
-
_record_worker_busy("epoch")
|
|
4152
|
-
return "busy"
|
|
4153
4591
|
# #997 D8: record the run (and any dead predecessor) before any work;
|
|
4154
4592
|
# a run that cannot be recorded does none.
|
|
4155
4593
|
run = _WorkerRun("epoch")
|
|
@@ -4162,11 +4600,7 @@ def _stats_epoch_rebuild_worker(args) -> str:
|
|
|
4162
4600
|
finally:
|
|
4163
4601
|
run.finish(outcome)
|
|
4164
4602
|
finally:
|
|
4165
|
-
|
|
4166
|
-
fcntl.flock(worker_fd, fcntl.LOCK_UN)
|
|
4167
|
-
except OSError:
|
|
4168
|
-
pass
|
|
4169
|
-
os.close(worker_fd)
|
|
4603
|
+
_release_rebuild_worker_claim(claim)
|
|
4170
4604
|
|
|
4171
4605
|
|
|
4172
4606
|
def _stats_epoch_rebuild_body() -> str:
|