cctally 1.116.1 → 1.117.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +57 -3
  4. package/bin/_cctally_cache.py +5720 -885
  5. package/bin/_cctally_claude_attribution.py +997 -0
  6. package/bin/_cctally_completion.py +101 -28
  7. package/bin/_cctally_core.py +235 -12
  8. package/bin/_cctally_dashboard.py +263 -17
  9. package/bin/_cctally_dashboard_conversation.py +107 -21
  10. package/bin/_cctally_dashboard_envelope.py +44 -0
  11. package/bin/_cctally_db.py +928 -80
  12. package/bin/_cctally_diagnosis_sources.py +78 -13
  13. package/bin/_cctally_doctor.py +211 -86
  14. package/bin/_cctally_five_hour.py +446 -22
  15. package/bin/_cctally_forecast.py +302 -6
  16. package/bin/_cctally_journal.py +2418 -158
  17. package/bin/_cctally_milestone_history.py +379 -91
  18. package/bin/_cctally_oauth_samples.py +185 -7
  19. package/bin/_cctally_parser.py +38 -0
  20. package/bin/_cctally_project.py +34 -8
  21. package/bin/_cctally_quarantine.py +821 -343
  22. package/bin/_cctally_quota.py +69 -4
  23. package/bin/_cctally_quota_calibration.py +677 -3
  24. package/bin/_cctally_quota_model.py +927 -127
  25. package/bin/_cctally_record.py +922 -445
  26. package/bin/_cctally_rederive.py +119 -15
  27. package/bin/_cctally_statusline.py +1048 -176
  28. package/bin/_cctally_store.py +544 -110
  29. package/bin/_cctally_tui.py +455 -42
  30. package/bin/_cctally_update.py +77 -14
  31. package/bin/_lib_account_protocol.py +227 -19
  32. package/bin/_lib_claude_attribution.py +421 -0
  33. package/bin/_lib_clone_copy.py +30 -0
  34. package/bin/_lib_codex_conversation_query.py +45 -9
  35. package/bin/_lib_codex_conversation_watch.py +72 -13
  36. package/bin/_lib_codex_rollout_container.py +234 -19
  37. package/bin/_lib_codex_worker_lifecycle.py +374 -0
  38. package/bin/_lib_credit.py +22 -13
  39. package/bin/_lib_dashboard_sources.py +257 -18
  40. package/bin/_lib_detector_access_registry.py +1335 -166
  41. package/bin/_lib_doctor.py +276 -92
  42. package/bin/_lib_ingest_frontier.py +269 -36
  43. package/bin/_lib_journal.py +273 -22
  44. package/bin/_lib_jsonl.py +12 -0
  45. package/bin/_lib_meter_rate_change.py +72 -0
  46. package/bin/_lib_outcome_quarantine.py +917 -554
  47. package/bin/_lib_provider_activity.py +24 -6
  48. package/bin/_lib_quota_calibration.py +57 -3
  49. package/bin/_lib_quota_copy.py +7 -2
  50. package/bin/_lib_quota_lineage.py +613 -0
  51. package/bin/_lib_quota_model.py +464 -33
  52. package/bin/_lib_rate_change_delivery.py +16 -0
  53. package/bin/_lib_reclaim_planner.py +6 -4
  54. package/bin/_lib_rederive.py +2046 -21
  55. package/bin/_lib_selector_state.py +131 -3
  56. package/bin/_lib_snapshot_cache.py +8 -0
  57. package/bin/_lib_stale_proof.py +113 -4
  58. package/bin/_lib_statusline.py +72 -3
  59. package/bin/_lib_statusline_candidates.py +246 -42
  60. package/bin/_lib_tick_stats.py +10 -5
  61. package/bin/_lib_write_io.py +419 -6
  62. package/bin/cctally +28 -11
  63. package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
  64. package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
  65. package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
  66. package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
  67. package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
  68. package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
  69. package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
  70. package/dashboard/static/assets/index-D34eYhAc.css +1 -0
  71. package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
  72. package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
  73. package/dashboard/static/dashboard.html +3 -3
  74. package/package.json +5 -1
  75. package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
  76. package/dashboard/static/assets/index-BKITk1CT.css +0 -1
@@ -1784,6 +1784,115 @@ def stats_open_guarded(
1784
1784
  lock_fh.close()
1785
1785
 
1786
1786
 
1787
+ #: #1015 spec 5.3: the fail-fast reader's SQLite busy timeout, in
1788
+ #: milliseconds — a contended read is "unavailable", never a wait.
1789
+ _STATS_FAILFAST_BUSY_TIMEOUT_MS = 50
1790
+
1791
+
1792
+ def _stats_failfast_epoch_probe(db_path: pathlib.Path) -> "int | None":
1793
+ """``PRAGMA user_version`` of the main file, read without locks or the
1794
+ wal-index (see `stats_open_readonly_failfast`); None on any error."""
1795
+ try:
1796
+ probe = sqlite3.connect(
1797
+ db_path.as_uri() + "?mode=ro&immutable=1", uri=True,
1798
+ **_STATS_CONNECT_KWARGS)
1799
+ except sqlite3.Error:
1800
+ return None
1801
+ try:
1802
+ return int(probe.execute("PRAGMA user_version").fetchone()[0])
1803
+ except (sqlite3.Error, TypeError, ValueError):
1804
+ return None
1805
+ finally:
1806
+ probe.close()
1807
+
1808
+
1809
+ def stats_open_readonly_failfast() -> "sqlite3.Connection | None":
1810
+ """A read-only connection to the PUBLISHED stats index, or None at once.
1811
+
1812
+ #1015 spec 5.3 (review findings R1-P1-4, R2-P1-4, R3-P1-1, R4-P2-3). For a
1813
+ reader that must never change, recover or wait on the stats family — the
1814
+ calibration lineage facts a forecast or the status line consults. It
1815
+ never calls `stats_open_guarded` (which resumes, resolves and recovers),
1816
+ `open_db`, a migration, ingest, a rebuild or a recovery callback.
1817
+
1818
+ 1. The maintenance flock is tried SHARED and NON-blocking; busy -> None.
1819
+ 2. None, touching nothing, when the repair marker, a pending quarantine
1820
+ record, an interrupted-rebuild artifact, the durable publication
1821
+ marker (pending or failed), or a ``-wal`` / ``-shm`` side file exists.
1822
+ Side files are found by ``stat`` only: no raw descriptor is ever opened
1823
+ on the database file, so this process's SQLite POSIX locks are never
1824
+ released by closing one, and the maintenance drain scanner never sees
1825
+ a descriptor this reader left open.
1826
+ 3. ``file:<path>?mode=ro`` as a URI, ``PRAGMA query_only = 1`` and a
1827
+ ``busy_timeout`` of `_STATS_FAILFAST_BUSY_TIMEOUT_MS`; None when
1828
+ ``PRAGMA user_version`` is not the current stats epoch, or on any
1829
+ SQLite error (a hot rollback journal surfaces as one).
1830
+ 4. The shared flock covers the checks and the connect and is released
1831
+ exactly as the guarded opener releases it, so the reader joins the
1832
+ #386 handshake: a destructive maintenance owner drains pre-marker
1833
+ handles.
1834
+
1835
+ R10 (a store last closed by a pre-#538 release keeps a WAL header with no
1836
+ side files) is narrowed by an epoch probe before the real connect: an
1837
+ ``immutable=1`` SQLite connection reads ``user_version`` from the main
1838
+ file without the wal-index, so an old-epoch store is refused before a
1839
+ read-only WAL open could create ``-shm`` (32 KiB) and ``-wal``. The probe
1840
+ is a fast reject only; the locked connection re-checks the epoch. It is
1841
+ an SQLite connection, not a raw descriptor: the unix VFS defers closing
1842
+ its descriptor while another connection of this process holds a lock on
1843
+ the same inode, so it releases no lock (the lock-preservation test pins
1844
+ this)."""
1845
+ db_path = pathlib.Path(_cctally_core.DB_PATH)
1846
+ lock_path = pathlib.Path(_cctally_core.STATS_LOCK_MAINTENANCE_PATH)
1847
+ try:
1848
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
1849
+ lock_fh = open(lock_path, "a+")
1850
+ except OSError:
1851
+ return None
1852
+ try:
1853
+ try:
1854
+ fcntl.flock(lock_fh, fcntl.LOCK_SH | fcntl.LOCK_NB)
1855
+ except OSError:
1856
+ return None
1857
+ try:
1858
+ if (_stats_repair_marker(db_path).exists()
1859
+ or _cctally_db._quarantine_pending_path(db_path).exists()
1860
+ or _stats_rebuild_artifact_bases(db_path)
1861
+ or _stats_publication_marker(db_path).exists()
1862
+ or pathlib.Path(str(db_path) + "-wal").exists()
1863
+ or pathlib.Path(str(db_path) + "-shm").exists()
1864
+ or not db_path.exists()):
1865
+ return None
1866
+ conn = None
1867
+ try:
1868
+ epoch = int(_cctally_core.STATS_INDEX_EPOCH)
1869
+ if _stats_failfast_epoch_probe(db_path) != epoch:
1870
+ return None
1871
+ conn = sqlite3.connect(
1872
+ db_path.as_uri() + "?mode=ro", uri=True,
1873
+ timeout=_STATS_FAILFAST_BUSY_TIMEOUT_MS / 1000.0,
1874
+ **_STATS_CONNECT_KWARGS)
1875
+ conn.execute("PRAGMA query_only = 1")
1876
+ # Literal SQL: `_STATS_FAILFAST_BUSY_TIMEOUT_MS` (50).
1877
+ conn.execute("PRAGMA busy_timeout = 50")
1878
+ version = conn.execute("PRAGMA user_version").fetchone()[0]
1879
+ if int(version) != epoch:
1880
+ conn.close()
1881
+ return None
1882
+ return conn
1883
+ except (sqlite3.Error, ValueError):
1884
+ if conn is not None:
1885
+ try:
1886
+ conn.close()
1887
+ except sqlite3.Error:
1888
+ pass
1889
+ return None
1890
+ finally:
1891
+ fcntl.flock(lock_fh, fcntl.LOCK_UN)
1892
+ finally:
1893
+ lock_fh.close()
1894
+
1895
+
1787
1896
  def _acquire_stats_maintenance_reentrant(path) -> "int | None":
1788
1897
  """Take ``stats.db.maintenance.lock`` EXCLUSIVE unless we already hold it.
1789
1898
 
@@ -2092,11 +2201,12 @@ def _stats_heal_hook(
2092
2201
  )
2093
2202
  if reservation == "reserved":
2094
2203
  active_request = elected_request or request
2095
- elif reservation == "pending":
2204
+ elif reservation in ("pending", "retry-elected"):
2096
2205
  # The elected request and coalesced count were read and updated
2097
2206
  # while admission was still held. Never reconstruct identity from
2098
2207
  # a marker that the worker may not have written yet or may already
2099
- # have removed.
2208
+ # have removed. `retry-elected` (#1012) keeps that request too: it
2209
+ # only adds a same-request contender.
2100
2210
  active_request = elected_request or request
2101
2211
  else:
2102
2212
  active_request = request
@@ -2219,6 +2329,11 @@ def _stats_heal_worker_path() -> pathlib.Path:
2219
2329
  return _stats_heal_path("stats-corruption-heal.worker.lock")
2220
2330
 
2221
2331
 
2332
+ def _stats_heal_claim_path() -> pathlib.Path:
2333
+ """#1012: taken only by heal workers, before the worker lock."""
2334
+ return _stats_heal_path("stats-corruption-heal.claim.lock")
2335
+
2336
+
2222
2337
  def _stats_heal_log_path() -> pathlib.Path:
2223
2338
  return pathlib.Path(_cctally_core.LOG_DIR) / "stats-corruption-heal.log"
2224
2339
 
@@ -2346,27 +2461,15 @@ def _unlink_stats_heal_marker() -> None:
2346
2461
 
2347
2462
 
2348
2463
  def _stats_heal_worker_active() -> bool:
2349
- """Probe the worker flock without waiting or disturbing its owner."""
2350
- try:
2351
- fd = os.open(
2352
- _stats_heal_worker_path(), os.O_WRONLY | os.O_CREAT, 0o600
2353
- )
2354
- except OSError:
2355
- return False
2356
- try:
2357
- try:
2358
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2359
- except BlockingIOError:
2360
- return True
2361
- except OSError:
2362
- return False
2363
- try:
2364
- fcntl.flock(fd, fcntl.LOCK_UN)
2365
- except OSError:
2366
- pass
2367
- return False
2368
- finally:
2369
- os.close(fd)
2464
+ """Whether a heal worker may be running, decided from its records.
2465
+
2466
+ #1012: true unless ownership is positively ``absent``, so a live heal
2467
+ whose identity cannot be confirmed keeps the maintenance-timeout message
2468
+ it had under the old lock probe. It takes no lock (neither the worker lock
2469
+ nor the claim lock) and writes nothing: the old try-lock, landing at a
2470
+ worker's claim instant, made that worker exit ``busy``.
2471
+ """
2472
+ return _rebuild_worker_ownership("heal") != "absent"
2370
2473
 
2371
2474
 
2372
2475
  def _log_stats_heal(
@@ -2461,14 +2564,30 @@ def _reserve_stats_corruption_heal(
2461
2564
  coalescedDetections=elected["coalescedDetections"],
2462
2565
  )
2463
2566
  return ("pending", elected)
2464
- if _stats_heal_worker_active():
2465
- # A real rebuild outlives the marker retry interval. Refresh the
2466
- # admission stamp instead of launching a process that can only lose
2467
- # the worker flock and exit.
2468
- elected = _read_stats_heal_request()
2469
- if not elected:
2567
+ ownership = _rebuild_worker_ownership("heal")
2568
+ kept = (_read_stats_heal_request()
2569
+ if ownership == "owned" or (ownership == "uncertain"
2570
+ and age is not None) else None)
2571
+ if ownership == "owned" or kept:
2572
+ # `owned`: a real rebuild outlives the marker retry interval.
2573
+ # Refresh the admission stamp instead of launching a process that
2574
+ # can only lose the claim and exit. `uncertain` (#1012): the heal
2575
+ # may still be live (a null recorded birth, a `ps` failure), and
2576
+ # replacing its request would make that worker's persist and
2577
+ # finalize refuse it. Keep the elected request, coalesce onto it,
2578
+ # and launch a same-request contender (`retry-elected`): the claim
2579
+ # protocol makes it hand off to a live owner or take over from a
2580
+ # dead one. With no marker, or an unreadable one, there is no
2581
+ # request to keep (a corrupt marker carries no healId a live
2582
+ # worker could still persist or finalize against), so an
2583
+ # uncertain record admits a new one below (its worker hands off
2584
+ # the same way); otherwise an unreadable state or marker would
2585
+ # fail every admission and nothing would ever launch the worker
2586
+ # that archives it. `owned` keeps failing on an unreadable
2587
+ # marker: its live worker still holds the request.
2588
+ if not kept:
2470
2589
  return ("failed", None)
2471
- elected = dict(elected)
2590
+ elected = dict(kept)
2472
2591
  elected["coalescedDetections"] = int(
2473
2592
  elected.get("coalescedDetections") or 0
2474
2593
  ) + 1
@@ -2481,7 +2600,8 @@ def _reserve_stats_corruption_heal(
2481
2600
  wait_s=_STATS_HEAL_ADMISSION_RING_WAIT_S,
2482
2601
  coalescedDetections=elected["coalescedDetections"],
2483
2602
  )
2484
- return ("pending", elected)
2603
+ return ("pending" if ownership == "owned" else "retry-elected",
2604
+ elected)
2485
2605
  stale_request = _read_stats_heal_request() if age is not None else None
2486
2606
  if stale_request:
2487
2607
  if not update_stats_heal_event(
@@ -2541,10 +2661,18 @@ def complete_stats_corruption_heal(reservation: str) -> str:
2541
2661
 
2542
2662
  Takes no lock: the spawn need not occur under any, and the maintenance hold
2543
2663
  must already have been released so the worker can take it exclusive. A
2544
- non-`reserved` reservation is returned unchanged, so a coalesced or failed
2545
- admission reports exactly what it reported before the split.
2664
+ reservation other than `reserved` or `retry-elected` is returned
2665
+ unchanged, so a coalesced or failed admission reports exactly what it
2666
+ reported before the split.
2667
+
2668
+ `retry-elected` (#1012) launches a contender for the EXISTING request of
2669
+ an owner whose identity could not be confirmed. It differs from
2670
+ `reserved` only on failure: the marker may belong to a live worker, so a
2671
+ failed spawn is recorded in the ledger but the marker is kept, and the
2672
+ result is `pending` (as is a withheld launch: launch capacity, or a
2673
+ contender already waiting on the same owner).
2546
2674
  """
2547
- if reservation != "reserved":
2675
+ if reservation not in ("reserved", "retry-elected"):
2548
2676
  return reservation
2549
2677
  # #997 D8 launch: the reservation is persisted under the admission flock
2550
2678
  # before the spawn, and the child's identity filled under it after. The
@@ -2557,14 +2685,22 @@ def complete_stats_corruption_heal(reservation: str) -> str:
2557
2685
  _with_heal_admission(lambda: reserve.append(_spawner_reserve(
2558
2686
  "heal", launch_id, _cctally_core.now_utc_iso())))
2559
2687
  if reserve == ["withheld"]:
2560
- # Launch capacity: the admitted request stays pending (its marker is
2561
- # kept) and nothing is spawned; a later detection retries it.
2688
+ # Launch capacity, or (#1012) a contender that has not started
2689
+ # already waits on an owner not proven gone: the admitted request
2690
+ # stays pending (its marker is kept) and nothing is spawned; a later
2691
+ # detection retries it.
2562
2692
  return "pending"
2563
2693
  pid = _spawn_rebuild_worker(STATS_CORRUPTION_HEAL_COMMAND, launch_id)
2564
2694
  if pid is not None:
2565
2695
  _with_heal_admission(lambda: _spawner_fill(
2566
2696
  "heal", launch_id, pid, _cctally_core.now_utc_iso()))
2567
2697
  return "spawned"
2698
+ if reservation == "retry-elected":
2699
+ # The elected request may still belong to a live worker: record the
2700
+ # failed start, keep its marker, and leave the request pending.
2701
+ _with_heal_admission(lambda: _spawner_fill(
2702
+ "heal", launch_id, None, _cctally_core.now_utc_iso()))
2703
+ return "pending"
2568
2704
  # Drop our own marker so the next detection is admitted immediately rather
2569
2705
  # than waiting out the retry window. Done under the admission flock, as it
2570
2706
  # was before the split, so no concurrent caller can coalesce onto a marker
@@ -2869,20 +3005,12 @@ def cmd_stats_corruption_heal_internal(args) -> int:
2869
3005
  """Hidden detached worker: heal one corrupt stats index exactly once."""
2870
3006
  del args
2871
3007
  healed = False
2872
- try:
2873
- pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True, exist_ok=True)
2874
- worker_fd = os.open(
2875
- _stats_heal_worker_path(), os.O_WRONLY | os.O_CREAT, 0o600
2876
- )
2877
- except OSError as exc:
2878
- _log_stats_heal("error", error=exc)
3008
+ # #1012: the claim lock, then the worker lock; a loser records `busy`
3009
+ # only against a live started owner (spec 5.3).
3010
+ claim = _claim_rebuild_worker("heal")
3011
+ if isinstance(claim, str):
2879
3012
  return 0
2880
3013
  try:
2881
- try:
2882
- fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2883
- except OSError:
2884
- _record_worker_busy("heal")
2885
- return 0
2886
3014
  # #997 D8: record the run (and any dead predecessor) before any work;
2887
3015
  # a run that cannot be recorded does none.
2888
3016
  run = _WorkerRun("heal")
@@ -2898,14 +3026,10 @@ def cmd_stats_corruption_heal_internal(args) -> int:
2898
3026
  finalization=result["finalization"],
2899
3027
  decline_reason=result["declineReason"])
2900
3028
  finally:
2901
- try:
2902
- fcntl.flock(worker_fd, fcntl.LOCK_UN)
2903
- except OSError:
2904
- pass
2905
- os.close(worker_fd)
3029
+ _release_rebuild_worker_claim(claim)
2906
3030
  if healed:
2907
3031
  # #825/#832 spec 4.3 revision 2: the healed index posts its
2908
- # completion wake, with the worker flock released.
3032
+ # completion wake, with the worker and claim locks released.
2909
3033
  import _cctally_completion
2910
3034
  _cctally_completion.post_publication_wake()
2911
3035
 
@@ -3197,6 +3321,11 @@ def _stats_epoch_rebuild_worker_path() -> pathlib.Path:
3197
3321
  return _stats_epoch_rebuild_path("stats-epoch-rebuild.worker.lock")
3198
3322
 
3199
3323
 
3324
+ def _stats_epoch_rebuild_claim_path() -> pathlib.Path:
3325
+ """#1012: taken only by epoch workers, before the worker lock."""
3326
+ return _stats_epoch_rebuild_path("stats-epoch-rebuild.claim.lock")
3327
+
3328
+
3200
3329
  def _stats_epoch_rebuild_log_path() -> pathlib.Path:
3201
3330
  return pathlib.Path(_cctally_core.LOG_DIR) / "stats-epoch-rebuild.log"
3202
3331
 
@@ -3209,29 +3338,16 @@ def _unlink_stats_epoch_marker() -> None:
3209
3338
 
3210
3339
 
3211
3340
  def _stats_epoch_rebuild_worker_active() -> bool:
3212
- """Probe the worker flock without waiting or disturbing its owner."""
3213
- try:
3214
- fd = os.open(
3215
- _stats_epoch_rebuild_worker_path(),
3216
- os.O_WRONLY | os.O_CREAT,
3217
- 0o600,
3218
- )
3219
- except OSError:
3220
- return False
3221
- try:
3222
- try:
3223
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
3224
- except BlockingIOError:
3225
- return True
3226
- except OSError:
3227
- return False
3228
- try:
3229
- fcntl.flock(fd, fcntl.LOCK_UN)
3230
- except OSError:
3231
- pass
3232
- return False
3233
- finally:
3234
- os.close(fd)
3341
+ """Whether a positively identified epoch worker owns the rebuild.
3342
+
3343
+ #1012: decided from the run state, launch ledger and process identity
3344
+ (``_rebuild_worker_ownership == "owned"``); an uncertain owner reads as
3345
+ inactive, so its caller launches a contender that the claim protocol makes
3346
+ hand off to a live owner. It takes no lock and writes nothing: the old
3347
+ try-lock of the worker flock, landing at a worker's claim instant, made
3348
+ that worker exit ``busy`` and left the pending rebuild unowned.
3349
+ """
3350
+ return _rebuild_worker_ownership("epoch") == "owned"
3235
3351
 
3236
3352
 
3237
3353
  def _log_stats_epoch_rebuild(outcome: str, *, error: Exception | None = None) -> None:
@@ -3315,6 +3431,18 @@ def _worker_admission_path(worker: str) -> pathlib.Path:
3315
3431
  return _stats_heal_admission_path()
3316
3432
 
3317
3433
 
3434
+ def _worker_claim_path(worker: str) -> pathlib.Path:
3435
+ if worker == "epoch":
3436
+ return _stats_epoch_rebuild_claim_path()
3437
+ return _stats_heal_claim_path()
3438
+
3439
+
3440
+ def _worker_lock_path(worker: str) -> pathlib.Path:
3441
+ if worker == "epoch":
3442
+ return _stats_epoch_rebuild_worker_path()
3443
+ return _stats_heal_worker_path()
3444
+
3445
+
3318
3446
  def _read_private_record(path: pathlib.Path):
3319
3447
  """``(payload | None, unreadable, raw bytes | None)`` for one record.
3320
3448
 
@@ -3440,12 +3568,13 @@ def read_stats_heal_request_snapshot() -> "dict | None":
3440
3568
  }
3441
3569
 
3442
3570
 
3443
- def process_identity(pid) -> dict:
3571
+ def process_identity(pid, *, timeout: float = 10) -> dict:
3444
3572
  """``{"status": "alive"|"gone"|"unknown", "birth": str | None}``.
3445
3573
 
3446
3574
  From ``ps -o stat=,lstart= -p <pid>`` under ``LC_ALL=C``: no such process
3447
3575
  (``ps`` exits 1 with no output) or a zombie (``Z`` in ``stat``) is
3448
- ``gone``; ``ps`` failing in any other way is ``unknown``. ``birth`` is the
3576
+ ``gone``; ``ps`` failing in any other way, including running past
3577
+ ``timeout`` seconds, is ``unknown``. ``birth`` is the
3449
3578
  one-second-resolution start time, whitespace-normalized.
3450
3579
  """
3451
3580
  import subprocess
@@ -3455,7 +3584,7 @@ def process_identity(pid) -> dict:
3455
3584
  try:
3456
3585
  out = subprocess.run(
3457
3586
  ["ps", "-o", "stat=,lstart=", "-p", str(pid)],
3458
- capture_output=True, text=True, timeout=10, check=False,
3587
+ capture_output=True, text=True, timeout=timeout, check=False,
3459
3588
  env={**os.environ, "LC_ALL": "C"})
3460
3589
  except (OSError, subprocess.SubprocessError):
3461
3590
  return {"status": "unknown", "birth": None}
@@ -3471,6 +3600,91 @@ def process_identity(pid) -> dict:
3471
3600
  return {"status": "alive", "birth": birth}
3472
3601
 
3473
3602
 
3603
+ #: #1012: the ownership check's own bound on its ``ps`` call, well below the
3604
+ #: diagnostics' 10 s. An identity that times out is ``unknown``, so the answer
3605
+ #: is ``uncertain``; an admission never stalls on ``ps``.
3606
+ _OWNERSHIP_IDENTITY_TIMEOUT_S = 2.0
3607
+
3608
+ #: Run fields a live worker rewrites as it progresses. A change to these alone
3609
+ #: between an ownership check's two reads is not a change of owner.
3610
+ _RUN_PROGRESS_FIELDS = frozenset({
3611
+ "phase", "phaseStartedAt", "progressAt", "counters", "residentBytes",
3612
+ "peakResidentBytes",
3613
+ })
3614
+
3615
+
3616
+ def _ownership_records(worker: str):
3617
+ """One read of the run state, then the ledger, for an ownership check.
3618
+
3619
+ ``(verdict, run, entry)``: ``verdict`` is ``"absent"`` or ``"uncertain"``
3620
+ when the records alone settle the answer, else ``None`` for a well-formed
3621
+ running run whose unresolved ledger entry acknowledges ``started`` with
3622
+ its ``runId``. Never writes.
3623
+ """
3624
+ state = read_worker_state(worker)
3625
+ if state["unreadable"]:
3626
+ return "uncertain", None, None
3627
+ run = (state["state"] or {}).get("run")
3628
+ if run is None or run.get("status") == "finished":
3629
+ return "absent", None, None
3630
+ pid = run.get("pid")
3631
+ if (run.get("status") != "running"
3632
+ or not isinstance(run.get("runId"), str)
3633
+ or not isinstance(run.get("launchId"), str)
3634
+ or not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0):
3635
+ return "uncertain", run, None
3636
+ ledger = read_launch_ledger(worker)
3637
+ if ledger["unreadable"]:
3638
+ return "uncertain", run, None
3639
+ entry = _wl.find(ledger["ledger"] or {}, run["launchId"])
3640
+ if entry is not None and entry.get("outcome"):
3641
+ # A recorded terminal is authoritative, as in `_reconcile_prior_run`.
3642
+ return "absent", run, entry
3643
+ ack = (entry or {}).get("ack")
3644
+ if (not isinstance(ack, dict) or ack.get("kind") != "started"
3645
+ or ack.get("runId") != run["runId"]):
3646
+ return "uncertain", run, entry
3647
+ return None, run, entry
3648
+
3649
+
3650
+ def _rebuild_worker_ownership(worker: str) -> str:
3651
+ """Who owns the worker's slot, from records alone (#1012).
3652
+
3653
+ ``"owned"`` only on positive evidence: a running run, its unresolved
3654
+ ledger entry acknowledging ``started`` with that ``runId``, and the run's
3655
+ pid alive with the recorded birth, confirmed by a second read of both
3656
+ records after the identity probe (progress-only updates are ignored).
3657
+ ``"absent"`` when no run is running, its entry is terminal, or its process
3658
+ is confirmed gone. ``"uncertain"`` otherwise: unreadable or malformed
3659
+ records, an entry that is missing or acknowledges another run, a null
3660
+ recorded birth, a ``ps`` failure or timeout, or a run that changed under
3661
+ the probe without finishing.
3662
+
3663
+ It never opens a lock file and never writes, so no check can make a
3664
+ starting worker lose its claim. Callers map the three answers (spec 5.2):
3665
+ an uncertain owner may never authorize a ``busy`` exit, and the heal
3666
+ admission never replaces an uncertain owner's request.
3667
+ """
3668
+ verdict, run, entry = _ownership_records(worker)
3669
+ if verdict is not None:
3670
+ return verdict
3671
+ pid = run["pid"]
3672
+ identity = process_identity(pid, timeout=_OWNERSHIP_IDENTITY_TIMEOUT_S)
3673
+ liveness = _wl.liveness(run, {pid: identity})
3674
+ if liveness == "gone":
3675
+ return "absent"
3676
+ if liveness != "alive":
3677
+ return "uncertain"
3678
+ again, run_again, entry_again = _ownership_records(worker)
3679
+ if again is not None:
3680
+ return again
3681
+ unchanged = (
3682
+ {k: v for k, v in run_again.items() if k not in _RUN_PROGRESS_FIELDS}
3683
+ == {k: v for k, v in run.items() if k not in _RUN_PROGRESS_FIELDS}
3684
+ and entry_again.get("ack") == entry.get("ack"))
3685
+ return "owned" if unchanged else "uncertain"
3686
+
3687
+
3474
3688
  def _archive_unreadable_record(path: pathlib.Path, raw) -> "str | None":
3475
3689
  """Copy an unreadable record's bytes to ``<name>.unreadable-<UTC stamp>``
3476
3690
  (a new private file, ``fsync``ed); the original stays in place until its
@@ -3541,23 +3755,56 @@ def _spawner_classify(worker: str, ledger: dict, now: str) -> dict:
3541
3755
  return _wl.classify(ledger, state, identities, now=now)["ledger"]
3542
3756
 
3543
3757
 
3758
+ def _contender_on_standby(ledger: dict, state, now: str) -> bool:
3759
+ """#1012: whether a contender already waits on an unsettled owner.
3760
+
3761
+ True when the run state's run is ``running`` and its launch is not
3762
+ settled in the classified ``ledger`` (an absent entry, or one with no
3763
+ outcome: the owner is not proven gone or finished), AND another
3764
+ unresolved launch has no acknowledgement (it has not started) and was
3765
+ reserved within ``_CONTENDER_STANDBY_S`` before ``now``. An
3766
+ unparseable reservation time never withholds."""
3767
+ run = state.get("run") if isinstance(state, dict) else None
3768
+ if not isinstance(run, dict) or run.get("status") != "running":
3769
+ return False
3770
+ own = run.get("launchId")
3771
+ entry = _wl.find(ledger, own) if isinstance(own, str) else None
3772
+ if entry is not None and entry.get("outcome"):
3773
+ return False
3774
+ now_at = _parse_utc_instant(now)
3775
+ if now_at is None:
3776
+ return False
3777
+ for other in _wl.unresolved(ledger):
3778
+ if other.get("launchId") == own or other.get("ack") is not None:
3779
+ continue
3780
+ at = _parse_utc_instant(other.get("at"))
3781
+ if at is not None and (0 <= (now_at - at).total_seconds()
3782
+ < _CONTENDER_STANDBY_S):
3783
+ return True
3784
+ return False
3785
+
3786
+
3544
3787
  def _spawner_reserve(worker: str, launch_id: str, now: str) -> str:
3545
3788
  """Classify the ledger and persist this launch's reservation (caller
3546
3789
  holds the admission flock).
3547
3790
 
3548
- ``"reserved"`` when the reservation is durable; ``"withheld"`` when
3791
+ ``"reserved"`` when the reservation is durable; ``"withheld"`` when the
3792
+ caller must not spawn (the settlements are written, no reservation is,
3793
+ and the request stays pending), for either of two reasons:
3549
3794
  ``LAUNCH_CAPACITY`` entries are still unresolved after classification
3550
- (the settlements are written, no reservation is, and the caller must not
3551
- spawn: the request stays pending and `doctor` reports
3552
- ``details.admission = "launch-capacity"``); ``"unwritten"`` when the
3553
- ledger could not be written, in which case the caller still spawns and
3554
- the worker's acknowledgement records the launch."""
3795
+ (`doctor` reports ``details.admission = "launch-capacity"``), or (#1012)
3796
+ a contender that has not started already waits on a recorded owner that
3797
+ is not proven gone or finished (``_contender_on_standby``), so a further
3798
+ one would only wait beside it; ``"unwritten"`` when the ledger could not
3799
+ be written, in which case the caller still spawns and the worker's
3800
+ acknowledgement records the launch."""
3555
3801
  try:
3556
3802
  ledger = _load_ledger_for_update(worker)
3557
3803
  ledger = _spawner_classify(worker, ledger, now)
3558
3804
  state = read_worker_state(worker)["state"]
3559
3805
  protected = _current_run_launch_id(state)
3560
- if not _wl.has_launch_capacity(ledger):
3806
+ if (not _wl.has_launch_capacity(ledger)
3807
+ or _contender_on_standby(ledger, state, now)):
3561
3808
  _write_ledger(worker, _wl.prune(ledger, protected))
3562
3809
  return "withheld"
3563
3810
  ledger = _wl.reserve(ledger, launch_id=launch_id, at=now)
@@ -3672,6 +3919,194 @@ def _record_worker_busy(worker: str) -> None:
3672
3919
  _heal_release_flock(fd)
3673
3920
 
3674
3921
 
3922
+ # --------------------------------------------------------------------------
3923
+ # #1012 — the claim protocol (spec 5.3)
3924
+ # --------------------------------------------------------------------------
3925
+ #
3926
+ # Only the two worker entry points take a claim lock: no probe, observer or
3927
+ # diagnostic may, or a check would again be able to defeat a starting worker.
3928
+ # A worker holds its claim lock for as long as it holds its worker lock, so a
3929
+ # claimant that finds the worker lock busy is looking at an observer (waited
3930
+ # out) or an older binary's worker (identified by its records: `busy`). The
3931
+ # one other owner its records can identify is a current worker after a double
3932
+ # fault: its `finish()` failed both the state write and the ledger settle,
3933
+ # and its process still lives.
3934
+ # Lock order: claim lock -> worker lock -> the admission / maintenance /
3935
+ # ingest order of docs/journal-gotchas.md.
3936
+
3937
+ #: A claimant retries a contended worker lock this often, with no elapsed-time
3938
+ #: bound: a non-owner hold delays the claimant and never defeats it.
3939
+ _CLAIM_RETRY_S = 0.02
3940
+ #: An admission launches no further contender while one that has not started
3941
+ #: (no acknowledgement) was reserved this recently for an owner not proven
3942
+ #: gone or finished (``_contender_on_standby``). That waiting contender's
3943
+ #: body starts after the current obligation and serves it, so a second would
3944
+ #: only wait beside it; a dead contender of unknown liveness delays a new
3945
+ #: launch by at most this long.
3946
+ _CONTENDER_STANDBY_S = 1800.0
3947
+ #: ...re-evaluating that lock's ownership at most this often (and at once on
3948
+ #: its first refusal). A claim loser's ownership check starts at this period.
3949
+ _CLAIM_OWNER_RECHECK_S = 1.0
3950
+ #: A claim loser sleeps a jittered first interval in this range, doubling to
3951
+ #: the cap; its ownership check backs off to once per the recheck cap.
3952
+ _HANDOFF_FIRST_SLEEP_S = (0.01, 0.03)
3953
+ _HANDOFF_SLEEP_CAP_S = 1.0
3954
+ _HANDOFF_RECHECK_CAP_S = 30.0
3955
+
3956
+
3957
+ def _unlock_close(fd: int) -> None:
3958
+ try:
3959
+ fcntl.flock(fd, fcntl.LOCK_UN)
3960
+ except OSError:
3961
+ pass
3962
+ finally:
3963
+ os.close(fd)
3964
+
3965
+
3966
+ def _hand_off_rebuild_worker_claim(worker: str, claim_fd: int) -> "str | None":
3967
+ """A claim loser's wait (spec 5.3 step 1): ``None`` once ``claim_fd``
3968
+ holds the claim; ``"busy"`` (after ``_record_worker_busy``) once a live
3969
+ worker that has started holds the worker lock; ``"error"`` on an I/O
3970
+ error that is not contention.
3971
+
3972
+ Only positive ownership authorizes ``busy``: a holder between its two
3973
+ flocks or before ``begin()``, a stale record and an unconfirmable
3974
+ identity all keep the loser waiting, so the pending work never loses its
3975
+ last claimant. Its worker-lock try is released at once, it holds no lock
3976
+ while it sleeps, and nothing bounds the wait but the holder's release.
3977
+ """
3978
+ import random
3979
+
3980
+ sleep_s = random.uniform(*_HANDOFF_FIRST_SLEEP_S)
3981
+ recheck_s = _CLAIM_OWNER_RECHECK_S
3982
+ next_check = None
3983
+ while True:
3984
+ try:
3985
+ worker_fd = os.open(_worker_lock_path(worker),
3986
+ os.O_WRONLY | os.O_CREAT, 0o600)
3987
+ except OSError as exc:
3988
+ _worker_log_error(worker, "claim", exc)
3989
+ return "error"
3990
+ try:
3991
+ try:
3992
+ fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
3993
+ except BlockingIOError:
3994
+ contended = True
3995
+ except OSError as exc:
3996
+ _worker_log_error(worker, "claim", exc)
3997
+ return "error"
3998
+ else:
3999
+ contended = False
4000
+ # Closing the descriptor releases the try anyway; an unlock
4001
+ # error must not escape the worker.
4002
+ try:
4003
+ fcntl.flock(worker_fd, fcntl.LOCK_UN)
4004
+ except OSError:
4005
+ pass
4006
+ finally:
4007
+ os.close(worker_fd)
4008
+ if contended:
4009
+ now = time.monotonic()
4010
+ if next_check is None or now >= next_check:
4011
+ if _rebuild_worker_ownership(worker) == "owned":
4012
+ _record_worker_busy(worker)
4013
+ return "busy"
4014
+ next_check = now + recheck_s
4015
+ recheck_s = min(recheck_s * 2, _HANDOFF_RECHECK_CAP_S)
4016
+ time.sleep(sleep_s)
4017
+ sleep_s = min(sleep_s * 2, _HANDOFF_SLEEP_CAP_S)
4018
+ try:
4019
+ fcntl.flock(claim_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
4020
+ except BlockingIOError:
4021
+ continue
4022
+ except OSError as exc:
4023
+ _worker_log_error(worker, "claim", exc)
4024
+ return "error"
4025
+ return None
4026
+
4027
+
4028
+ def _claim_rebuild_worker(worker: str) -> "tuple[int, int] | str":
4029
+ """Take the worker's claim lock, then its worker lock (spec 5.3).
4030
+
4031
+ Returns the held ``(claim_fd, worker_fd)`` for
4032
+ ``_release_rebuild_worker_claim``, or ``"busy"`` (recorded through
4033
+ ``_record_worker_busy``) or ``"error"`` (logged ``stage=claim``, with no
4034
+ acknowledgement). A claim loser hands off before any ``busy``; a claimant
4035
+ waits out any hold of the worker lock except a positively identified
4036
+ owner. While this process holds the claim that owner is an older
4037
+ binary's worker, or, after a double fault, a current one whose
4038
+ ``finish()`` failed both its state write and its ledger settle while its
4039
+ process lives on: its stale ``running`` record then names a live pid.
4040
+ """
4041
+ fds: list = [] # open descriptors not handed to the caller, claim first
4042
+
4043
+ def release_all() -> None:
4044
+ while fds:
4045
+ _unlock_close(fds.pop()) # the worker lock first, then the claim
4046
+
4047
+ def failed(exc) -> str:
4048
+ release_all()
4049
+ _worker_log_error(worker, "claim", exc)
4050
+ return "error"
4051
+
4052
+ try:
4053
+ try:
4054
+ pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True,
4055
+ exist_ok=True)
4056
+ fds.append(os.open(_worker_claim_path(worker),
4057
+ os.O_WRONLY | os.O_CREAT, 0o600))
4058
+ except OSError as exc:
4059
+ return failed(exc)
4060
+ claim_fd = fds[0]
4061
+ try:
4062
+ fcntl.flock(claim_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
4063
+ except BlockingIOError:
4064
+ handed = _hand_off_rebuild_worker_claim(worker, claim_fd)
4065
+ if handed is not None:
4066
+ return handed
4067
+ except OSError as exc:
4068
+ return failed(exc)
4069
+ try:
4070
+ fds.append(os.open(_worker_lock_path(worker),
4071
+ os.O_WRONLY | os.O_CREAT, 0o600))
4072
+ except OSError as exc:
4073
+ return failed(exc)
4074
+ worker_fd = fds[1]
4075
+ next_check = None
4076
+ while True:
4077
+ try:
4078
+ fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
4079
+ except BlockingIOError:
4080
+ pass
4081
+ except OSError as exc:
4082
+ return failed(exc)
4083
+ else:
4084
+ fds.clear()
4085
+ return (claim_fd, worker_fd)
4086
+ now = time.monotonic()
4087
+ if next_check is None or now >= next_check:
4088
+ next_check = now + _CLAIM_OWNER_RECHECK_S
4089
+ if _rebuild_worker_ownership(worker) == "owned":
4090
+ release_all()
4091
+ _record_worker_busy(worker)
4092
+ return "busy"
4093
+ time.sleep(_CLAIM_RETRY_S)
4094
+ finally:
4095
+ release_all()
4096
+
4097
+
4098
+ def _release_rebuild_worker_claim(claim: "tuple[int, int]") -> None:
4099
+ """Release a held claim: the worker lock first, then the claim lock, and
4100
+ both before any completion wake. Releasing the claim first would let a
4101
+ second claimant hold it while this process still held the worker lock,
4102
+ and misread this process as an older binary's owner."""
4103
+ claim_fd, worker_fd = claim
4104
+ try:
4105
+ _unlock_close(worker_fd)
4106
+ finally:
4107
+ _unlock_close(claim_fd)
4108
+
4109
+
3675
4110
  def rebuild_worker_snapshot(worker: str) -> dict:
3676
4111
  """What `doctor` and `db status` classify, gathered without any lock
3677
4112
  and without writing (spec D8 "Liveness without the flock").
@@ -4060,8 +4495,10 @@ def defer_stats_epoch_rebuild() -> str:
4060
4495
  # launch id, then fill the child's identity (or record the failure).
4061
4496
  launch_id = _new_launch_id()
4062
4497
  if _spawner_reserve("epoch", launch_id, now) == "withheld":
4063
- # Launch capacity: the marker (just refreshed, so the retry
4064
- # throttle holds) keeps the request pending; nothing is spawned.
4498
+ # Launch capacity, or (#1012) a contender that has not started
4499
+ # already waits on an owner not proven gone: the marker (just
4500
+ # refreshed, so the retry throttle holds) keeps the request
4501
+ # pending; nothing is spawned.
4065
4502
  return "pending"
4066
4503
  pid = _spawn_rebuild_worker(STATS_EPOCH_REBUILD_COMMAND, launch_id)
4067
4504
  _spawner_fill("epoch", launch_id, pid, _cctally_core.now_utc_iso())
@@ -4083,7 +4520,18 @@ def stats_rebuild_worker_blocks() -> tuple:
4083
4520
  `_cctally_journal.LockBlock` naming the worker lock that refused it,
4084
4521
  whether it was held (contention) or the probe hit an I/O error. A worker
4085
4522
  lock that is free, or whose file does not exist, yields nothing: an
4086
- unlocked probe proves no release."""
4523
+ unlocked probe proves no release.
4524
+
4525
+ #1012: this stays lock-based, unlike the ownership probes, which never
4526
+ acquire a worker lock. Actual contention (including the window before a
4527
+ worker's run record exists) is what chooses between a release obligation
4528
+ and an inline rebuild, and a stale record must not invent an obligation.
4529
+ This function and the statusline dependency observers take only the
4530
+ worker (compatibility) lock, never a claim lock; the claim protocol keeps
4531
+ their brief holds from defeating a claimant, and a claim loser returns
4532
+ `busy` only while a positively identified live worker holds the worker
4533
+ lock this function observes (a holder may still release between that
4534
+ `busy` and this probe: spec residual R5)."""
4087
4535
  import _cctally_journal
4088
4536
 
4089
4537
  blocks = []
@@ -4134,22 +4582,12 @@ STATS_EPOCH_RESIDUAL_NOT_DURABLE = "success-residual-not-durable"
4134
4582
  def _stats_epoch_rebuild_worker(args) -> str:
4135
4583
  """The worker body; returns its outcome."""
4136
4584
  del args
4585
+ # #1012: the claim lock, then the worker lock; `busy` only against a live
4586
+ # started owner, `error` for a lock I/O failure (spec 5.3).
4587
+ claim = _claim_rebuild_worker("epoch")
4588
+ if isinstance(claim, str):
4589
+ return claim
4137
4590
  try:
4138
- pathlib.Path(_cctally_core.APP_DIR).mkdir(parents=True, exist_ok=True)
4139
- worker_fd = os.open(
4140
- _stats_epoch_rebuild_worker_path(),
4141
- os.O_WRONLY | os.O_CREAT,
4142
- 0o600,
4143
- )
4144
- except OSError as exc:
4145
- _log_stats_epoch_rebuild("error", error=exc)
4146
- return "error"
4147
- try:
4148
- try:
4149
- fcntl.flock(worker_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
4150
- except OSError:
4151
- _record_worker_busy("epoch")
4152
- return "busy"
4153
4591
  # #997 D8: record the run (and any dead predecessor) before any work;
4154
4592
  # a run that cannot be recorded does none.
4155
4593
  run = _WorkerRun("epoch")
@@ -4162,11 +4600,7 @@ def _stats_epoch_rebuild_worker(args) -> str:
4162
4600
  finally:
4163
4601
  run.finish(outcome)
4164
4602
  finally:
4165
- try:
4166
- fcntl.flock(worker_fd, fcntl.LOCK_UN)
4167
- except OSError:
4168
- pass
4169
- os.close(worker_fd)
4603
+ _release_rebuild_worker_claim(claim)
4170
4604
 
4171
4605
 
4172
4606
  def _stats_epoch_rebuild_body() -> str: