cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -679,15 +679,23 @@ def _read_db_projection_once() -> "_candidates.DbProjection":
679
679
  _acct_pred = "" if _sl_account is None else " AND account_key = ?"
680
680
  _acct_params: tuple = () if _sl_account is None else (_sl_account,)
681
681
  try:
682
+ # `weekly_observation_held = 0` (#769 S11, #824): the WEEKLY half of the
683
+ # projection. The rendered percentage and the capture instant that
684
+ # decides which candidate wins both come from the row selected here, so
685
+ # a held row would project an older reading as this tick's observation.
682
686
  weekly_rows = conn.execute(
683
687
  "SELECT id, weekly_percent, week_start_date, week_start_at, week_end_at, "
684
688
  " captured_at_utc, source "
685
689
  "FROM weekly_usage_snapshots "
686
690
  "WHERE weekly_percent IS NOT NULL AND week_end_at IS NOT NULL"
691
+ " AND weekly_observation_held = 0"
687
692
  + _acct_pred +
688
693
  " ORDER BY unixepoch(captured_at_utc) DESC, id DESC",
689
694
  _acct_params,
690
695
  ).fetchall()
696
+ # HELD-INCLUSIVE, deliberately: the FIVE-HOUR half. A held row exists
697
+ # because its five-hour reading grew, so dropping it here would hold
698
+ # the status line's five-hour percentage at the last unclamped tick.
691
699
  five_rows = conn.execute(
692
700
  "SELECT id, five_hour_percent, five_hour_resets_at, five_hour_window_key, "
693
701
  " captured_at_utc, source "
@@ -850,13 +858,34 @@ def _pending_document(value: "_candidates.PendingDrop | None") -> dict | None:
850
858
  "dbResetGeneration": signature.db_reset_generation,
851
859
  }
852
860
  ),
861
+ "deadlineAt": value.deadline_at,
862
+ "retained": (
863
+ None if value.retained is None else {
864
+ "percent": value.retained.percent,
865
+ "rawResetsAt": value.retained.raw_resets_at,
866
+ "canonicalKey": value.retained.canonical_key,
867
+ "observations": [
868
+ {
869
+ "observationId": item.observation_id,
870
+ "token": item.token,
871
+ "percent": item.percent,
872
+ "rawResetsAt": item.raw_resets_at,
873
+ "receivedAt": item.received_at,
874
+ }
875
+ for item in value.retained.observations
876
+ ],
877
+ }
878
+ ),
853
879
  }
854
880
 
855
881
 
856
882
  def _control_document(control: "_candidates.ControlState") -> dict:
857
883
  files = control.db_projection.db_files or {"main": None, "wal": None}
858
884
  return {
859
- "schemaVersion": 1,
885
+ # Versioned independently of the candidate and tombstone documents, so
886
+ # that a control shape change cannot invalidate the spool documents a
887
+ # peer session running the previous binary is still writing.
888
+ "schemaVersion": _candidates.CONTROL_SCHEMA_VERSION,
860
889
  "dbProjection": {
861
890
  "fiveHour": _axis_projection_document(control.db_projection.five_hour),
862
891
  "sevenDay": _axis_projection_document(control.db_projection.seven_day),
@@ -1181,16 +1210,15 @@ def _statusline_reduce_and_publish(
1181
1210
  """
1182
1211
  now_epoch = int(time.time())
1183
1212
  candidates = _load_candidate_spool(now_epoch=now_epoch)
1184
- if not candidates:
1185
- existing = _read_control_state(now_epoch=now_epoch)
1186
- if existing is not None and any(existing.pending_drops.values()):
1187
- projection = _read_db_projection_stable()
1188
- control = _empty_control(projection)
1189
- _write_control_state(control)
1190
- return _candidates.ReductionDecision("WRITE_CONTROL", control)
1213
+ existing = _read_control_state(now_epoch=now_epoch)
1214
+ if not candidates and (existing is None or not any(existing.pending_drops.values())):
1191
1215
  return None
1216
+ # An empty spool is NOT a retraction (#755). Every contributor supporting a
1217
+ # pending low can age out of its 90-second active window well before that
1218
+ # drop's own 180-second deadline, so the reducer runs over the empty
1219
+ # candidate set and lets the pending record decide from its own retained
1220
+ # evidence. Discarding the record here is what the production stall did.
1192
1221
  projection = _read_db_projection_stable()
1193
- existing = _read_control_state(now_epoch=now_epoch)
1194
1222
  control_repair_required = (
1195
1223
  existing is None or existing.db_projection.db_files != projection.db_files
1196
1224
  )
@@ -1727,6 +1755,9 @@ def _build_statusline_injections(warn_once):
1727
1755
  if five_resets is not None:
1728
1756
  try:
1729
1757
  key = c._canonical_5h_window_key(int(five_resets))
1758
+ # HELD-INCLUSIVE (#769 S11, #824): a five-hour maximum over
1759
+ # a five-hour window, and a held row's five-hour value is
1760
+ # the writing tick's own.
1730
1761
  row = conn.execute(
1731
1762
  "SELECT MAX(five_hour_percent) "
1732
1763
  "FROM weekly_usage_snapshots "
@@ -1777,11 +1808,22 @@ def _build_statusline_injections(warn_once):
1777
1808
  week_start_dt.isoformat(), week_end_dt.isoformat(),
1778
1809
  account_key=_sl_account,
1779
1810
  )
1811
+ # `weekly_observation_held = 0` (#769 S11, #824): a WEEKLY
1812
+ # maximum, and the floored leg is why it matters. A held
1813
+ # row written after a credit floor carries the PRE-credit
1814
+ # value forward under a post-floor capture instant, so
1815
+ # counting it would re-clamp the status line to a
1816
+ # high-water mark the credit retired — the same defence
1817
+ # `_resolve_reset_aware_hwm` carries, restated here because
1818
+ # this is an independent implementation of that maximum.
1819
+ # Both legs take the predicate: the two must not disagree
1820
+ # about what a held row is.
1780
1821
  if floor_iso is not None:
1781
1822
  row = conn.execute(
1782
1823
  "SELECT MAX(weekly_percent) "
1783
1824
  "FROM weekly_usage_snapshots "
1784
1825
  "WHERE week_start_date = ?" + _acct_pred + " "
1826
+ " AND weekly_observation_held = 0 "
1785
1827
  " AND unixepoch(captured_at_utc) >= unixepoch(?)",
1786
1828
  (week_start_date, *_acct_params, floor_iso),
1787
1829
  ).fetchone()
@@ -1789,7 +1831,8 @@ def _build_statusline_injections(warn_once):
1789
1831
  row = conn.execute(
1790
1832
  "SELECT MAX(weekly_percent) "
1791
1833
  "FROM weekly_usage_snapshots "
1792
- "WHERE week_start_date = ?" + _acct_pred,
1834
+ "WHERE week_start_date = ?" + _acct_pred +
1835
+ " AND weekly_observation_held = 0",
1793
1836
  (week_start_date, *_acct_params),
1794
1837
  ).fetchone()
1795
1838
  if row and row[0] is not None:
@@ -1816,20 +1859,41 @@ def _build_statusline_injections(warn_once):
1816
1859
  # may have `week_end_at` NULL — fall back to the date column
1817
1860
  # in that case. See the neighbor query in `pick_week_selection`
1818
1861
  # (bin/cctally:3849) for the precedent.
1819
- row = conn.execute(
1820
- "SELECT five_hour_percent, five_hour_window_key, "
1821
- " weekly_percent, week_end_at, week_end_date "
1862
+ #
1863
+ # TWO queries, one per axis (#769 S11, #824). This used to be one
1864
+ # row because one row carried both readings, and a
1865
+ # `weekly_observation_held` row is the case where it does not: its
1866
+ # five-hour fields are this tick's own while its weekly value and
1867
+ # boundary were copied from an older row. The five-hour leg takes
1868
+ # the newest row of either kind; the weekly leg takes the newest
1869
+ # genuinely observed one. Asking one row for both would either hold
1870
+ # the five-hour percentage back or present a carried-forward weekly
1871
+ # percentage as this tick's reading.
1872
+ five_row = conn.execute(
1873
+ "SELECT five_hour_percent, five_hour_window_key "
1874
+ "FROM weekly_usage_snapshots "
1875
+ "ORDER BY captured_at_utc DESC, id DESC LIMIT 1"
1876
+ ).fetchone()
1877
+ weekly_row = conn.execute(
1878
+ "SELECT weekly_percent, week_end_at, week_end_date "
1822
1879
  "FROM weekly_usage_snapshots "
1880
+ "WHERE weekly_observation_held = 0 "
1823
1881
  "ORDER BY captured_at_utc DESC, id DESC LIMIT 1"
1824
1882
  ).fetchone()
1825
- if not row:
1883
+ if not five_row and not weekly_row:
1826
1884
  return None
1827
- five_pct = float(row[0]) if row[0] is not None else None
1828
- five_resets = int(row[1]) if row[1] is not None else None
1829
- seven_pct = float(row[2]) if row[2] is not None else None
1885
+ five_pct = (
1886
+ float(five_row[0])
1887
+ if five_row and five_row[0] is not None else None)
1888
+ five_resets = (
1889
+ int(five_row[1])
1890
+ if five_row and five_row[1] is not None else None)
1891
+ seven_pct = (
1892
+ float(weekly_row[0])
1893
+ if weekly_row and weekly_row[0] is not None else None)
1830
1894
  seven_resets = None
1831
- week_end_at = row[3]
1832
- week_end_date = row[4]
1895
+ week_end_at = weekly_row[1] if weekly_row else None
1896
+ week_end_date = weekly_row[2] if weekly_row else None
1833
1897
  if week_end_at:
1834
1898
  try:
1835
1899
  # `datetime.fromisoformat` accepts the trailing `Z`
@@ -285,6 +285,44 @@ def schema_current(conn: sqlite3.Connection, store: str) -> bool:
285
285
  return user_version == head
286
286
 
287
287
 
288
+ def schema_state(conn: sqlite3.Connection, store: str, *,
289
+ schema: str = "main") -> str:
290
+ """``"current"`` / ``"behind"`` / ``"ahead"`` for ``store`` under ``schema``.
291
+
292
+ The read-only conversation opener (#780) needs two things ``schema_current``
293
+ structurally cannot give it. It must address an ATTACHED store, and
294
+ ``schema_current`` always runs an unqualified ``PRAGMA user_version``, which
295
+ reports ``main`` however the attachment is stamped. And it must tell BEHIND
296
+ from AHEAD, because the two have opposite remedies: behind wakes a writer
297
+ and fails soft, while ahead fails closed with no recovery attempt, matching
298
+ the existing version-ahead posture. A bare boolean collapses both into
299
+ "not current".
300
+
301
+ ``schema`` is a SQLite schema name, not user input — it is a literal at
302
+ every call site, and it is validated against the connection's own
303
+ ``database_list`` before being interpolated, because ``PRAGMA`` does not
304
+ take a bound parameter for its schema qualifier.
305
+
306
+ A store with no registry head to gate on reports ``"behind"``: the soft
307
+ direction, and the same conservative answer ``schema_current`` gives with
308
+ its bare ``False``. The opener never migrates from a request thread
309
+ whichever answer it gets.
310
+ """
311
+ if schema != "main":
312
+ attached = {
313
+ row[1] for row in conn.execute("PRAGMA database_list")
314
+ }
315
+ if schema not in attached:
316
+ raise ValueError(f"schema {schema!r} is not attached")
317
+ head = _expected_head(store)
318
+ if head <= 0:
319
+ return "behind"
320
+ user_version = conn.execute(f"PRAGMA {schema}.user_version").fetchone()[0]
321
+ if user_version == head:
322
+ return "current"
323
+ return "behind" if user_version < head else "ahead"
324
+
325
+
288
326
  # --------------------------------------------------------------------------
289
327
  # §7.1 stats.db epoch gate (Task 9)
290
328
  # --------------------------------------------------------------------------
@@ -1372,6 +1410,25 @@ def _recover_or_reclaim_interrupted_stats_rebuild(
1372
1410
  _cctally_journal._release_ingest_lock(ingest_fd)
1373
1411
 
1374
1412
 
1413
+ #: #778 — every guarded stats connection is built with the statement cache OFF.
1414
+ #:
1415
+ #: `set_authorizer` installs a callback SQLite invokes while it COMPILES a
1416
+ #: statement, never while it runs one, and Python's `sqlite3` caches 128
1417
+ #: compiled statements per connection. `arm_stats_authorizer` answers from the
1418
+ #: `stats_write_scope` `ContextVar`, so a statement first prepared inside a
1419
+ #: sanctioned scope is re-used verbatim afterwards and the guard is never
1420
+ #: consulted again — the same SQL executed after the scope exits, or from a
1421
+ #: thread whose `ContextVar` was never set, wrote unchallenged. With no cache
1422
+ #: every `execute()` re-prepares, so authorization is decided per execution.
1423
+ #:
1424
+ #: Passed to the caller-supplied `connect` callable as a MANDATORY keyword. A
1425
+ #: connector that cannot accept it raises `TypeError`, and that call is never
1426
+ #: retried without the keyword: the loud failure is what makes a future
1427
+ #: connection factory that forgets to forward `**kwargs` a test failure rather
1428
+ #: than a silently unguarded connection.
1429
+ _STATS_CONNECT_KWARGS = {"cached_statements": 0}
1430
+
1431
+
1375
1432
  def stats_open_guarded(
1376
1433
  db_path=None, *, connect=None, recover_interruptions: bool = True
1377
1434
  ) -> sqlite3.Connection:
@@ -1384,8 +1441,8 @@ def stats_open_guarded(
1384
1441
 
1385
1442
  ``connect`` lets a caller keep its own open mode (``mode=ro`` for
1386
1443
  ``db backup``, ``mode=rw`` for ``db status``) while still participating; it
1387
- receives the path and returns the connection. Defaults to
1388
- ``sqlite3.connect``.
1444
+ receives the path and ``cached_statements=0`` (see ``_STATS_CONNECT_KWARGS``)
1445
+ and returns the connection. Defaults to ``sqlite3.connect``.
1389
1446
  """
1390
1447
  db_path = pathlib.Path(
1391
1448
  db_path if db_path is not None else _cctally_core.DB_PATH
@@ -1399,7 +1456,7 @@ def stats_open_guarded(
1399
1456
  # Pre-#386 behaviour, unchanged.
1400
1457
  if marker.exists():
1401
1458
  raise _cctally_db.StatsDbMaintenanceError()
1402
- conn = _connect(db_path)
1459
+ conn = _connect(db_path, **_STATS_CONNECT_KWARGS)
1403
1460
  arm_stats_authorizer(conn)
1404
1461
  return conn
1405
1462
 
@@ -1537,7 +1594,7 @@ def stats_open_guarded(
1537
1594
  fcntl.flock(lock_fh, fcntl.LOCK_UN)
1538
1595
  continue
1539
1596
  try:
1540
- conn = _connect(db_path)
1597
+ conn = _connect(db_path, **_STATS_CONNECT_KWARGS)
1541
1598
  # Re-check inside the same shared hold: cheap, and it closes the
1542
1599
  # window between the checks above and a slow connect.
1543
1600
  if marker.exists() or pending.exists():
@@ -116,6 +116,18 @@ def _cmd_transcript_export(args) -> int:
116
116
  return 0
117
117
 
118
118
 
119
+ #: The M5 refusal, verbatim. Exit 3 is the code `docs/commands/transcript.md`
120
+ #: already uses for an attribution cache that cannot be established.
121
+ _ANON_UNAVAILABLE_MSG = (
122
+ "transcript: anonymized export is unavailable: {count} Codex project "
123
+ "path(s) could not be read; run cctally cache-sync --source codex --rebuild"
124
+ )
125
+ _ANON_AMBIGUOUS_MSG = (
126
+ "transcript: anonymized export is unavailable: {count} Codex project "
127
+ "path(s) have ambiguous account provenance; use --raw or omit --account"
128
+ )
129
+
130
+
119
131
  def _cmd_transcript_export_qualified(
120
132
  c, args, session_id, scope, raw, output, speed_arg) -> int:
121
133
  """Qualified (``v1.``) export via the neutral dispatch layer (§4.1). Anonymized
@@ -157,10 +169,23 @@ def _cmd_transcript_export_qualified(
157
169
  if not raw:
158
170
  cq = c._load_sibling("_lib_conversation_query")
159
171
  anon = c._load_sibling("_lib_conversation_anon")
160
- plan = cq.build_anon_plan_for_sources(
172
+ result = cq.build_anon_plan_for_sources(
161
173
  conn, home_dir=os.path.expanduser("~"),
162
174
  sources={cref.source})
163
- md = anon.scrub_text(md, plan)
175
+ # #850 M5. A Codex project path the store cannot read cannot become
176
+ # a scrub token, and such a path can appear in ANY transcript, so
177
+ # the anonymized export refuses while any of them is undecodable
178
+ # rather than emitting bytes that only look scrubbed. `--raw` and
179
+ # `search` escape, and a Claude-only plan never reaches this.
180
+ if result.undecodable_cwd_rows:
181
+ eprint(_ANON_UNAVAILABLE_MSG.format(
182
+ count=result.undecodable_cwd_rows))
183
+ return 3
184
+ if result.ambiguous_cwd_rows:
185
+ eprint(_ANON_AMBIGUOUS_MSG.format(
186
+ count=result.ambiguous_cwd_rows))
187
+ return 3
188
+ md = anon.scrub_text(md, result.plan)
164
189
  finally:
165
190
  conn.close()
166
191