cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -47,6 +47,7 @@ import collections
47
47
  import dataclasses
48
48
  import hashlib
49
49
  import secrets
50
+ import select
50
51
  import threading
51
52
  import time
52
53
 
@@ -115,10 +116,16 @@ _OUTLINE_TRANSFER_ITEM_CAP = 32 * 1024 * 1024
115
116
  _OUTLINE_TRANSFER_TOTAL_CAP = 64 * 1024 * 1024
116
117
  _OUTLINE_TRANSFER_COUNT_CAP = 128
117
118
  _OUTLINE_TRANSFER_TTL = 120.0
119
+ _OUTLINE_TRANSFER_REQUEST_DEADLINE = 15.0
120
+ _OUTLINE_TRANSFER_MAX_WAITERS = 32
121
+ _OUTLINE_TRANSFER_MAX_WAITERS_PER_TOKEN = 16
122
+ _OUTLINE_TRANSFER_MAX_PRODUCERS = 2
118
123
  _OUTLINE_TRANSFERS = collections.OrderedDict()
119
124
  _OUTLINE_TRANSFERS_BYTES = 0
120
125
  _OUTLINE_TRANSFERS_EVICTIONS = 0
121
126
  _OUTLINE_TRANSFERS_FALLBACKS = 0
127
+ _OUTLINE_TRANSFERS_WAITERS = 0
128
+ _OUTLINE_TRANSFERS_PRODUCERS = 0
122
129
  _OUTLINE_TRANSFERS_LOCK = threading.Lock()
123
130
 
124
131
 
@@ -129,6 +136,10 @@ class _OutlineTransferRecord:
129
136
  payload: bytes | None = None
130
137
  digest: str | None = None
131
138
  state: str = "pending"
139
+ waiters: int = 0
140
+ deadline: float | None = None
141
+ failure_status: int | None = None
142
+ failure_body: dict | None = None
132
143
  cancelled: threading.Event = dataclasses.field(default_factory=threading.Event)
133
144
  ready: threading.Event = dataclasses.field(default_factory=threading.Event)
134
145
 
@@ -148,6 +159,8 @@ def outline_transfer_cache_stats():
148
159
  "maxEntries": _OUTLINE_TRANSFER_COUNT_CAP,
149
160
  "evictionCount": int(_OUTLINE_TRANSFERS_EVICTIONS),
150
161
  "fallbackCount": int(_OUTLINE_TRANSFERS_FALLBACKS),
162
+ "admittedWaiters": int(_OUTLINE_TRANSFERS_WAITERS),
163
+ "activeProducers": int(_OUTLINE_TRANSFERS_PRODUCERS),
151
164
  }
152
165
 
153
166
 
@@ -191,8 +204,9 @@ def _store_outline_transfer(builder):
191
204
  return {"token": token, "chunk_size": _OUTLINE_TRANSFER_CHUNK}
192
205
 
193
206
 
194
- def _claim_outline_transfer(token):
207
+ def _claim_outline_transfer(token, *, admit=False):
195
208
  """Atomically claim a pending builder or join its in-flight generation."""
209
+ global _OUTLINE_TRANSFERS_WAITERS, _OUTLINE_TRANSFERS_PRODUCERS
196
210
  now = time.monotonic()
197
211
  with _OUTLINE_TRANSFERS_LOCK:
198
212
  _prune_outline_transfers_locked(now)
@@ -202,12 +216,49 @@ def _claim_outline_transfer(token):
202
216
  _OUTLINE_TRANSFERS.move_to_end(token)
203
217
  if record.payload is not None:
204
218
  return "ready", record, None
219
+ if admit:
220
+ if (_OUTLINE_TRANSFERS_WAITERS >= _OUTLINE_TRANSFER_MAX_WAITERS
221
+ or record.waiters >= _OUTLINE_TRANSFER_MAX_WAITERS_PER_TOKEN
222
+ or (record.state == "pending" and
223
+ _OUTLINE_TRANSFERS_PRODUCERS >=
224
+ _OUTLINE_TRANSFER_MAX_PRODUCERS)):
225
+ return "busy", record, None
226
+ record.waiters += 1
227
+ _OUTLINE_TRANSFERS_WAITERS += 1
205
228
  if record.state == "pending":
206
229
  record.state = "building"
230
+ if admit:
231
+ record.deadline = min(
232
+ now + _OUTLINE_TRANSFER_REQUEST_DEADLINE,
233
+ record.created + _OUTLINE_TRANSFER_TTL,
234
+ )
235
+ _OUTLINE_TRANSFERS_PRODUCERS += 1
207
236
  return "build", record, record.builder
208
237
  return "wait", record, None
209
238
 
210
239
 
240
+ def _release_outline_transfer_waiter(token, record):
241
+ """An abandoned last reader cancels its producer without releasing its slot."""
242
+ global _OUTLINE_TRANSFERS_WAITERS
243
+ with _OUTLINE_TRANSFERS_LOCK:
244
+ record.waiters -= 1
245
+ _OUTLINE_TRANSFERS_WAITERS -= 1
246
+ if (record.waiters == 0 and record.state == "building"
247
+ and _OUTLINE_TRANSFERS.get(token) is record):
248
+ _drop_outline_transfer_locked(token)
249
+
250
+
251
+ def _finish_outline_transfer_failure(token, record, status, body):
252
+ """Publish one failure to admitted readers and retire only this generation."""
253
+ with _OUTLINE_TRANSFERS_LOCK:
254
+ if _OUTLINE_TRANSFERS.get(token) is not record:
255
+ return False
256
+ record.failure_status = status
257
+ record.failure_body = body
258
+ _drop_outline_transfer_locked(token)
259
+ return True
260
+
261
+
211
262
  def _outline_transfer_builder(token):
212
263
  """Compatibility probe: claim one builder, ``False`` if joined/ready."""
213
264
  claim = _claim_outline_transfer(token)
@@ -225,6 +276,9 @@ def _materialize_outline_transfer(token, payload, *, claim=None):
225
276
  with _OUTLINE_TRANSFERS_LOCK:
226
277
  current = _OUTLINE_TRANSFERS.get(token)
227
278
  if current is not None and (claim is None or current is claim):
279
+ current.failure_status = 413
280
+ current.failure_body = {
281
+ "error": "outline exceeds progressive transfer limit"}
228
282
  _drop_outline_transfer_locked(token)
229
283
  _OUTLINE_TRANSFERS_FALLBACKS += 1
230
284
  return "too_large"
@@ -266,6 +320,9 @@ def _materialize_outline_transfer(token, payload, *, claim=None):
266
320
  None,
267
321
  )
268
322
  if victim is None:
323
+ record.failure_status = 413
324
+ record.failure_body = {
325
+ "error": "outline exceeds progressive transfer limit"}
269
326
  _drop_outline_transfer_locked(token)
270
327
  _OUTLINE_TRANSFERS_FALLBACKS += 1
271
328
  return "too_large"
@@ -292,6 +349,9 @@ def _reject_outline_transfer_too_large(token, *, claim) -> bool:
292
349
  current = _OUTLINE_TRANSFERS.get(token)
293
350
  if current is None or current is not claim:
294
351
  return False
352
+ current.failure_status = 413
353
+ current.failure_body = {
354
+ "error": "outline exceeds progressive transfer limit"}
295
355
  _drop_outline_transfer_locked(token)
296
356
  _OUTLINE_TRANSFERS_FALLBACKS += 1
297
357
  return True
@@ -655,8 +715,13 @@ def _handle_qualified_facets(handler, qs_raw, source):
655
715
  if not ok:
656
716
  return
657
717
  facets = (body or {}).get("facets") or {"projects": [], "models": []}
658
- handler._respond_json(
659
- 200, {"status": (body or {}).get("status"), "facets": facets})
718
+ # #717: the SECOND reshaping site. This rebuilds the response from two
719
+ # named keys, so an unknown key added upstream reaches no client unless it
720
+ # is carried here too.
721
+ payload = {"status": (body or {}).get("status"), "facets": facets}
722
+ if (body or {}).get("filter_degraded"):
723
+ payload["filter_degraded"] = True
724
+ handler._respond_json(200, payload)
660
725
 
661
726
 
662
727
  def _handle_qualified_search(handler, qs_raw, source):
@@ -712,6 +777,156 @@ def _parse_search_kind_impl(handler, q, valid=_CONV_SEARCH_KINDS):
712
777
  return None
713
778
  return kind
714
779
 
780
+ #: #780 — how many times a read route re-attempts admission before it degrades.
781
+ #: Bounded by count, never by wall clock: the opener's own busy timeout is
782
+ #: already route-bounded, and an unbounded retry against a rebuild that holds
783
+ #: the store for minutes would pin a request thread for the whole rebuild.
784
+ _CONVERSATION_READ_ATTEMPTS = 3
785
+ _CONVERSATION_READ_RETRY_SLEEP_S = 0.02
786
+
787
+ #: The empty-but-valid body each named read route serves alongside the typed
788
+ #: degraded marker. Keyed on the caller's own log label, so no call site
789
+ #: changes and a route with no entry still gets the marker.
790
+ _CONVERSATION_DEGRADED_SHAPES = {
791
+ "/api/conversations": {"conversations": [], "total": 0},
792
+ "/api/conversations/facets": {"projects": [], "models": []},
793
+ "/api/conversation/search": {"results": [], "total": 0},
794
+ "/api/conversation/find": {"matches": [], "total": 0},
795
+ "/api/conversation/prompts": {"prompts": [], "total": 0},
796
+ "/api/conversation": {},
797
+ "/api/conversation/outline": {},
798
+ "/api/conversation/payload": {},
799
+ "/api/conversation/media": {},
800
+ "/api/conversation/export": {},
801
+ # The live-tail preflight answers before any SSE byte is committed, so its
802
+ # refusal is plain JSON like every other route's. An empty shape is right
803
+ # here: the client's own backstop tick is what recovers, and there is no
804
+ # partial stream to render.
805
+ "/api/conversation/events": {},
806
+ }
807
+
808
+
809
+ def _reader_unavailable():
810
+ """The typed reader-admission condition, resolved lazily.
811
+
812
+ `_cctally_dashboard` re-exports the cache module's opener, and the tests
813
+ monkeypatch that re-export, so the class is read through the same module
814
+ object the opener came from rather than imported at module scope.
815
+ """
816
+ return sys.modules["_cctally_dashboard"].ConversationReaderUnavailable
817
+
818
+
819
+ def _conversation_degraded_body(log_label, reason):
820
+ """One typed degraded envelope for a read route that could not be admitted.
821
+
822
+ HTTP 200, because the condition is transient and expected: maintenance is
823
+ running, or a writer has not yet advanced the schema. It carries the
824
+ route's own empty shape so a client that ignores the marker renders an
825
+ empty surface rather than failing to parse, and it names no SQLite text, no
826
+ SQL and no filesystem path — the privacy posture the 500 envelope already
827
+ held.
828
+ """
829
+ body = dict(_CONVERSATION_DEGRADED_SHAPES.get(log_label, {}))
830
+ body["status"] = "degraded"
831
+ body["degraded_reason"] = reason
832
+ return body
833
+
834
+
835
+ def _full_open_or_refuse():
836
+ """Open write-capable, and refuse if the legacy bridge is STILL owed (#802).
837
+
838
+ The full opener is what fixes an interrupted migration 028: it runs
839
+ `_import_legacy_conversation_rows`, which moves the transcript rows out of
840
+ `cache.db` and derives the browse rollup from them. Under `dashboard
841
+ --no-sync` the process policy suppresses that derivation, so this opener
842
+ returns a store where the rows are still in `cache.db` and the rollup is
843
+ still empty — and `list_conversation_facets` reads the rollup's
844
+ `project_label` with no authoritative gate, so the conversation list and
845
+ the browse project filter would both render empty for the life of the
846
+ process, with no pending state and nothing said.
847
+
848
+ Spec §9 permits exactly this alternative for a route that cannot inherit
849
+ the derivation policy: return a typed degraded response instead. The
850
+ outcome is checked rather than the policy, so a bridge that COMPLETED is
851
+ byte-unchanged, and a future third reason the bridge could not run is
852
+ covered without a second predicate.
853
+
854
+ The claim is about the outcome, not about the mode. It was first written as
855
+ "every mode allowed to do work is byte-unchanged", and that is too strong:
856
+ `_import_legacy_conversation_rows` is fail-soft per table (`except
857
+ sqlite3.Error: continue`), so a mode that IS allowed to do work but whose
858
+ import raised on one table leaves the predicate true and is refused here,
859
+ where the old code returned the connection. That refusal is the better
860
+ answer — the old code served an empty conversation surface over rows it had
861
+ just failed to move, and said nothing about it — but it is a change, so it
862
+ is stated rather than covered by an absolute.
863
+ """
864
+ dash = sys.modules["_cctally_dashboard"]
865
+ cache_mod = sys.modules["_cctally_cache"]
866
+ conn = dash.open_conversations_db()
867
+ if not cache_mod.conversation_legacy_bridge_pending(conn):
868
+ return conn
869
+ conn.close()
870
+ raise dash.LegacyBridgePending(
871
+ "the legacy transcript bridge is owed and derivations are suppressed")
872
+
873
+
874
+ def _open_conversation_reader(log_label):
875
+ """Admit one read route, with a bounded retry (#780).
876
+
877
+ Reads through the read-only opener, which performs no schema apply, no
878
+ migration dispatch, no PRAGMA write and no chmod, so a browse is no longer
879
+ a writer that can lose the SQLite write lock to a reclaim pass.
880
+
881
+ A store that is simply not present yet falls back to the full opener, which
882
+ creates and migrates it. That is the first-run path, not the steady state,
883
+ and removing it would make an empty installation serve a degraded envelope
884
+ forever with no process willing to create the store.
885
+
886
+ BOTH fallbacks probe the maintenance flock first. The full opener's
887
+ admission is a plain `LOCK_SH` with no timeout, so falling back to it
888
+ during a rebuild queued the request behind the whole rebuild — 716.9 s in
889
+ the measured run — which is the failure this opener exists to remove,
890
+ reintroduced on the two paths that leave it. The probe refuses instead, and
891
+ the route serves the same typed degraded envelope every other refusal
892
+ already produces. The legacy-bridge fallback matters more than the
893
+ first-run one here: `conversation_legacy_bridge_pending` is durable state
894
+ after an interrupted migration 028, so it is not a one-off startup case.
895
+
896
+ Both fallbacks go through `_full_open_or_refuse`, which serves the opened
897
+ connection only when the bridge is no longer owed. Under `--no-sync` the
898
+ bridge is suppressed and cannot clear, so the route answers the typed
899
+ degraded envelope rather than an empty conversation surface (#802).
900
+ """
901
+ dash = sys.modules["_cctally_dashboard"]
902
+ cache_mod = sys.modules["_cctally_cache"]
903
+ last = None
904
+ for attempt in range(_CONVERSATION_READ_ATTEMPTS):
905
+ try:
906
+ conn = dash.open_conversations_db_readonly()
907
+ except dash.MaintenanceInProgress as exc:
908
+ last = exc
909
+ if attempt + 1 < _CONVERSATION_READ_ATTEMPTS:
910
+ time.sleep(_CONVERSATION_READ_RETRY_SLEEP_S)
911
+ continue
912
+ raise
913
+ except dash.ConversationReaderUnavailable as exc:
914
+ if type(exc) is dash.ConversationReaderUnavailable:
915
+ # The store (or the accounting store it attaches) is absent.
916
+ dash.probe_conversations_maintenance_free()
917
+ return _full_open_or_refuse()
918
+ raise
919
+ if cache_mod.conversation_legacy_bridge_pending(conn):
920
+ # An interrupted migration 028 left transcript rows in cache.db.
921
+ # The bridge that moves them is a writer, so hand the open back to
922
+ # the full opener rather than serving an empty surface over rows
923
+ # that are present but not yet in this store.
924
+ conn.close()
925
+ dash.probe_conversations_maintenance_free()
926
+ return _full_open_or_refuse()
927
+ return conn
928
+ raise last # unreachable: the loop either returns or raises
929
+
715
930
  def _run_conversation_query_impl(
716
931
  handler, kernel_call, log_label, *, cancelled=None,
717
932
  ):
@@ -728,7 +943,17 @@ def _run_conversation_query_impl(
728
943
  ``{type}: {msg}`` 500.
729
944
  """
730
945
  try:
731
- conn = sys.modules["_cctally_dashboard"].open_conversations_db()
946
+ conn = _open_conversation_reader(log_label)
947
+ except _reader_unavailable() as exc:
948
+ # #780: maintenance and schema admission are NOT server errors. A
949
+ # rebuild, a reclaim pass or a store one migration behind used to reach
950
+ # the reader as a 500, which is exactly the 5xx this session exists to
951
+ # remove. Answer with the route's own typed degraded envelope instead.
952
+ handler.log_error(
953
+ "%s transcript store degraded: %s", log_label, exc.reason)
954
+ handler._respond_json(200, _conversation_degraded_body(
955
+ log_label, exc.reason))
956
+ return False, None
732
957
  except (sqlite3.DatabaseError, OSError) as exc:
733
958
  handler.log_error("%s transcript store open failed: %r", log_label, exc)
734
959
  handler._respond_json(
@@ -1021,7 +1246,9 @@ def _run_conversation_events_stream(
1021
1246
  if passive:
1022
1247
  # Frozen-data contract: no ingest, no emit. Keep-alive only.
1023
1248
  while True:
1024
- _time.sleep(_LIVE_TAIL_KEEPALIVE)
1249
+ if _request_peer_gone(
1250
+ handler, timeout=_LIVE_TAIL_KEEPALIVE):
1251
+ return
1025
1252
  handler.wfile.write(b": keep-alive\n\n")
1026
1253
  handler.wfile.flush()
1027
1254
 
@@ -1056,7 +1283,8 @@ def _run_conversation_events_stream(
1056
1283
  idle = 0.0
1057
1284
  cycles = 0
1058
1285
  while True:
1059
- _time.sleep(_LIVE_TAIL_POLL_INTERVAL)
1286
+ if _request_peer_gone(handler, timeout=_LIVE_TAIL_POLL_INTERVAL):
1287
+ return
1060
1288
  cycles += 1
1061
1289
  changed = watch.changed_paths(files, seen)
1062
1290
  if changed:
@@ -1103,6 +1331,62 @@ def _run_conversation_events_stream(
1103
1331
  idle = 0.0
1104
1332
 
1105
1333
 
1334
+ def _live_tail_read_connection():
1335
+ """The READ connection a live-tail stream holds for its whole lifetime.
1336
+
1337
+ Deliberately the full opener, not `open_conversations_db_readonly`, and the
1338
+ reason is measured rather than assumed. Moving it to the read-only opener
1339
+ made `test_codex_child_discovery_emits_tail` stop emitting a tail: the
1340
+ stream reached `ready` and `baselined`, the child rollout was ingested
1341
+ through the separate writer below, and the long-lived read connection never
1342
+ observed the committed growth, so `seen` never changed and no `event: tail`
1343
+ was ever pushed.
1344
+
1345
+ That is a different problem from the one #780 is about. A read ROUTE opens
1346
+ per request, so its opener's write-capable pragmas are what compete with
1347
+ maintenance for the SQLite write lock on every browse; a live-tail stream
1348
+ opens ONCE and then holds the connection for minutes, so it contributes one
1349
+ open, not one per request. Task 3.3's requirement — that every live-tail
1350
+ INGEST branch takes its own write connection — is met by `_live_tail_write`
1351
+ below, and that is the part that would otherwise make each stream a writer.
1352
+
1353
+ What did NOT survive is entering that opener unconditionally. Its
1354
+ maintenance admission is a blocking `LOCK_SH`, so a stream opened during a
1355
+ rebuild waited out the rebuild — 716.9 s measured — holding a server thread
1356
+ the whole time and answering nothing. §4a names the live-tail preflight
1357
+ among the routes that must fail soft, so the flock is probed non-blocking
1358
+ first and the two callers turn `MaintenanceInProgress` into the same typed
1359
+ degraded envelope every other read route serves.
1360
+
1361
+ It goes through `_full_open_or_refuse` for the same reason the read routes
1362
+ do: under `--no-sync` the legacy bridge is suppressed, so a stream opened
1363
+ over an owed bridge would tail a store whose transcript rows are still in
1364
+ `cache.db` and report nothing for as long as it ran (#802).
1365
+ """
1366
+ dash = sys.modules["_cctally_dashboard"]
1367
+ dash.probe_conversations_maintenance_free()
1368
+ return _full_open_or_refuse()
1369
+
1370
+
1371
+ def _live_tail_write(work):
1372
+ """Run one live-tail ingest on its OWN write connection (#780).
1373
+
1374
+ Live-tail is the single write exception among the conversation routes.
1375
+ Its own read connection is NOT read-only — `_live_tail_read_connection`
1376
+ above keeps the full opener, and says why — but that connection must still
1377
+ not be the one that ingests: a stream holds it for minutes, and driving a
1378
+ sync on it mixes the stream's long-lived read transaction with writes. The
1379
+ ingest branches therefore open a writer, use it, and close it. The stream's
1380
+ read connection is refreshed by the caller's next query, which starts a
1381
+ fresh read transaction and so observes what this writer committed.
1382
+ """
1383
+ writer = sys.modules["_cctally_dashboard"].open_conversations_db()
1384
+ try:
1385
+ return work(writer)
1386
+ finally:
1387
+ writer.close()
1388
+
1389
+
1106
1390
  def _bare_conversation_events(
1107
1391
  handler, session_id: str, account_key: str | None = None,
1108
1392
  ) -> None:
@@ -1111,7 +1395,16 @@ def _bare_conversation_events(
1111
1395
  cq = handler._conversation_query()
1112
1396
  passive = bool(type(handler).no_sync)
1113
1397
  try:
1114
- conn = sys.modules["_cctally_dashboard"].open_conversations_db()
1398
+ conn = _live_tail_read_connection()
1399
+ except _reader_unavailable() as exc:
1400
+ # #780: maintenance is not a server error and not a passive stream
1401
+ # either. Answer the typed degraded envelope BEFORE any SSE byte, the
1402
+ # same way every other read route does.
1403
+ handler._respond_json(
1404
+ 200, _conversation_degraded_body(
1405
+ "/api/conversation/events", getattr(exc, "reason",
1406
+ "unavailable")))
1407
+ return
1115
1408
  except (sqlite3.DatabaseError, OSError):
1116
1409
  if account_key is not None:
1117
1410
  # A qualified account boundary must be established before any SSE
@@ -1139,7 +1432,14 @@ def _bare_conversation_events(
1139
1432
  def _ingest(changed):
1140
1433
  _advance_live_tail_accounting(handler, changed, sync_cache, "claude")
1141
1434
  if account_key is None:
1142
- return sync_claude_conversations(conn, only_paths=set(changed))
1435
+ # #780: ingest takes its OWN write connection. The read connection
1436
+ # this stream then queries is opened read-only, so writing through
1437
+ # it would fail outright — and even on a writable connection,
1438
+ # driving ingestion from the connection a long-lived SSE stream
1439
+ # reads from is what made every live-tail a writer.
1440
+ return _live_tail_write(
1441
+ lambda writer: sync_claude_conversations(
1442
+ writer, only_paths=set(changed)))
1143
1443
  before = conn.execute(
1144
1444
  "SELECT COUNT(*),MAX(id) FROM conversation_messages "
1145
1445
  "WHERE session_id=?",
@@ -1250,8 +1550,22 @@ def _qualified_conversation_events(
1250
1550
  targeted ingest + the directory-frontier child discovery."""
1251
1551
  disp = _conversation_dispatch_impl()
1252
1552
  try:
1253
- conn = sys.modules["_cctally_dashboard"].open_conversations_db()
1553
+ conn = _live_tail_read_connection()
1554
+ except _reader_unavailable() as exc:
1555
+ # #780: maintenance holds the store. The preflight has committed no SSE
1556
+ # bytes yet, so this is answerable as the same typed degraded envelope
1557
+ # every other read route serves — and it must be, because a passive
1558
+ # stream here would look to the client like a conversation with no
1559
+ # turns rather than a store that is busy.
1560
+ handler._respond_json(
1561
+ 200, _conversation_degraded_body(
1562
+ "/api/conversation/events", getattr(exc, "reason",
1563
+ "unavailable")))
1564
+ return
1254
1565
  except (sqlite3.DatabaseError, OSError):
1566
+ # Any other open failure takes the passive keep-alive stream: a
1567
+ # genuinely unopenable store — an incomplete recovery, a live repair
1568
+ # marker, an I/O error.
1255
1569
  conn = None
1256
1570
 
1257
1571
  if conn is None:
@@ -1310,7 +1624,9 @@ def _qualified_conversation_events(
1310
1624
  if account_key is None:
1311
1625
  _advance_live_tail_accounting(
1312
1626
  handler, changed, sync_codex_cache, "codex")
1313
- return sync_codex_conversations(conn, only_paths=set(changed))
1627
+ return _live_tail_write(
1628
+ lambda writer: sync_codex_conversations(
1629
+ writer, only_paths=set(changed)))
1314
1630
  before = conn.execute(
1315
1631
  "SELECT COUNT(*),MAX(id) FROM codex_conversation_messages "
1316
1632
  "WHERE conversation_key=?",
@@ -1356,7 +1672,9 @@ def _qualified_conversation_events(
1356
1672
  _advance_live_tail_accounting(
1357
1673
  handler, changed, sync_cache, "claude")
1358
1674
  if account_key is None:
1359
- return sync_claude_conversations(conn, only_paths=set(changed))
1675
+ return _live_tail_write(
1676
+ lambda writer: sync_claude_conversations(
1677
+ writer, only_paths=set(changed)))
1360
1678
  before = conn.execute(
1361
1679
  "SELECT COUNT(*),MAX(id) FROM conversation_messages "
1362
1680
  "WHERE session_id=?",
@@ -1538,7 +1856,8 @@ def _handle_get_conversation_payload_impl(handler, path: str) -> None:
1538
1856
 
1539
1857
  def _handle_get_conversation_outline_impl(handler, path: str) -> None:
1540
1858
  """``GET /api/conversation/<sid>/outline`` — full-session skeleton +
1541
- session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404.
1859
+ session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404
1860
+ for the legacy form, typed absence on the progressive preflight.
1542
1861
  """
1543
1862
  if not handler._require_transcripts_allowed():
1544
1863
  return
@@ -1604,7 +1923,7 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
1604
1923
  if not ok:
1605
1924
  return
1606
1925
  if not exists:
1607
- handler.send_error(404, "conversation not found")
1926
+ handler._respond_json(200, {"status": "not_found"})
1608
1927
  return
1609
1928
 
1610
1929
  def _build_bare(transfer_handler, cancelled):
@@ -1637,6 +1956,99 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
1637
1956
  handler._respond_json(200, body)
1638
1957
 
1639
1958
 
1959
+ class _OutlineTransferBuildHandler:
1960
+ """Give a producer read access to its request without writing its socket."""
1961
+
1962
+ def __init__(self, handler):
1963
+ self._handler = handler
1964
+ self.response = None
1965
+
1966
+ def __getattr__(self, name):
1967
+ return getattr(self._handler, name)
1968
+
1969
+ def _respond_json(self, status, body):
1970
+ self.response = (status, body)
1971
+
1972
+
1973
+ def _produce_outline_transfer(token, record, builder, handler):
1974
+ """Build off the request thread; the producer slot lasts until real exit."""
1975
+ global _OUTLINE_TRANSFERS_PRODUCERS
1976
+ proxy = _OutlineTransferBuildHandler(handler)
1977
+ try:
1978
+ try:
1979
+ ok, full_body = builder(proxy, record.cancelled.is_set)
1980
+ if record.cancelled.is_set():
1981
+ return
1982
+ if ok is None or full_body is None and ok:
1983
+ _finish_outline_transfer_failure(
1984
+ token, record, 410,
1985
+ {"error": "outline transfer expired"})
1986
+ return
1987
+ if not ok:
1988
+ status, body = proxy.response or (
1989
+ 500, {"error": "outline transfer failed"})
1990
+ _finish_outline_transfer_failure(token, record, status, body)
1991
+ return
1992
+ payload = encode_dashboard_json_bytes_capped(
1993
+ full_body, max_bytes=_OUTLINE_TRANSFER_ITEM_CAP)
1994
+ if payload is None:
1995
+ _reject_outline_transfer_too_large(token, claim=record)
1996
+ else:
1997
+ _materialize_outline_transfer(token, payload, claim=record)
1998
+ except Exception as exc: # noqa: BLE001
1999
+ handler.log_error("outline transfer build failed: %r", exc)
2000
+ _finish_outline_transfer_failure(
2001
+ token, record, 500, {"error": "outline transfer failed"})
2002
+ finally:
2003
+ with _OUTLINE_TRANSFERS_LOCK:
2004
+ _OUTLINE_TRANSFERS_PRODUCERS -= 1
2005
+
2006
+
2007
+ def _request_peer_gone(handler, *, timeout=0.0):
2008
+ """Detect a closed request socket while no response is being written."""
2009
+ conn = getattr(handler, "connection", None)
2010
+ if conn is None:
2011
+ return False
2012
+ try:
2013
+ if not select.select([conn], [], [], timeout)[0]:
2014
+ return False
2015
+ return not conn.recv(
2016
+ 1, socket.MSG_PEEK | getattr(socket, "MSG_DONTWAIT", 0))
2017
+ except (OSError, ValueError):
2018
+ return True
2019
+
2020
+
2021
+ def _wait_outline_transfer(handler, token, record):
2022
+ """Wait only to the shared build deadline, waking for expiry/disconnect."""
2023
+ while True:
2024
+ if _request_peer_gone(handler):
2025
+ return False
2026
+ if record.ready.is_set():
2027
+ if record.failure_status is not None:
2028
+ handler._respond_json(
2029
+ record.failure_status, record.failure_body)
2030
+ return False
2031
+ if record.cancelled.is_set() or record.payload is None:
2032
+ handler._respond_json(
2033
+ 410, {"error": "outline transfer expired"})
2034
+ return False
2035
+ return True
2036
+ now = time.monotonic()
2037
+ if now - record.created > _OUTLINE_TRANSFER_TTL:
2038
+ _cancel_outline_transfer(token, claim=record)
2039
+ handler._respond_json(410, {"error": "outline transfer expired"})
2040
+ return False
2041
+ if now >= record.deadline:
2042
+ _finish_outline_transfer_failure(
2043
+ token, record, 504,
2044
+ {"error": "outline transfer timed out"})
2045
+ handler._respond_json(
2046
+ record.failure_status or 410,
2047
+ record.failure_body or {"error": "outline transfer expired"})
2048
+ return False
2049
+ record.ready.wait(min(0.05, record.deadline - now))
2050
+
2051
+
1640
2052
  def _handle_get_conversation_outline_transfer_impl(handler, path: str) -> None:
1641
2053
  """Serve one bounded chunk from an opaque, immutable outline transfer."""
1642
2054
  if not handler._require_transcripts_allowed():
@@ -1655,42 +2067,33 @@ def _handle_get_conversation_outline_transfer_impl(handler, path: str) -> None:
1655
2067
  return
1656
2068
  else:
1657
2069
  offset = int(values[0])
1658
- claim = _claim_outline_transfer(token)
2070
+ claim = _claim_outline_transfer(token, admit=True)
1659
2071
  if claim is None:
1660
2072
  handler._respond_json(410, {"error": "outline transfer expired"})
1661
2073
  return
1662
2074
  action, record, builder = claim
2075
+ if action == "busy":
2076
+ handler._respond_json(503, {"error": "outline transfer busy"})
2077
+ return
1663
2078
  if action == "build":
1664
- ok, full_body = builder(handler, record.cancelled.is_set)
1665
- if ok is None:
1666
- handler._respond_json(410, {"error": "outline transfer expired"})
1667
- return
1668
- if not ok:
1669
- _cancel_outline_transfer(token, claim=record)
1670
- return
1671
- if full_body is None:
1672
- _cancel_outline_transfer(token, claim=record)
1673
- handler._respond_json(410, {"error": "outline transfer expired"})
1674
- return
1675
- payload = encode_dashboard_json_bytes_capped(
1676
- full_body, max_bytes=_OUTLINE_TRANSFER_ITEM_CAP)
1677
- if payload is None:
1678
- _reject_outline_transfer_too_large(token, claim=record)
1679
- state = "too_large"
1680
- else:
1681
- state = _materialize_outline_transfer(token, payload, claim=record)
1682
- if state == "too_large":
1683
- handler._respond_json(
1684
- 413, {"error": "outline exceeds progressive transfer limit"})
1685
- return
1686
- if state == "expired":
1687
- handler._respond_json(410, {"error": "outline transfer expired"})
1688
- return
1689
- elif action == "wait":
1690
- record.ready.wait(_OUTLINE_TRANSFER_TTL)
1691
- if record.cancelled.is_set() or record.payload is None:
1692
- handler._respond_json(410, {"error": "outline transfer expired"})
1693
- return
2079
+ try:
2080
+ threading.Thread(
2081
+ target=_produce_outline_transfer,
2082
+ args=(token, record, builder, handler),
2083
+ name="cctally-outline-build", daemon=True,
2084
+ ).start()
2085
+ except RuntimeError:
2086
+ with _OUTLINE_TRANSFERS_LOCK:
2087
+ global _OUTLINE_TRANSFERS_PRODUCERS
2088
+ _OUTLINE_TRANSFERS_PRODUCERS -= 1
2089
+ _finish_outline_transfer_failure(
2090
+ token, record, 503, {"error": "outline transfer busy"})
2091
+ if action != "ready":
2092
+ try:
2093
+ if not _wait_outline_transfer(handler, token, record):
2094
+ return
2095
+ finally:
2096
+ _release_outline_transfer_waiter(token, record)
1694
2097
  try:
1695
2098
  body = _read_outline_transfer(token, offset)
1696
2099
  except ValueError:
@@ -1749,6 +2152,33 @@ def _handle_get_conversation_prompts_impl(handler, path: str) -> None:
1749
2152
 
1750
2153
  _CONV_EXPORT_SCOPES = ("all", "prompts", "chat", "recipe")
1751
2154
 
2155
+ #: The #850 M5 fail-closed body, shared by `/export?anonymize=1` and
2156
+ #: `/anon-map`. Both routes answer HTTP 409 with it while any Codex thread
2157
+ #: `cwd` is undecodable: a path the store cannot supply as a scrub token can
2158
+ #: appear in any transcript, so neither route may hand back bytes or a token
2159
+ #: map that only looks scrubbed.
2160
+ _ANON_UNAVAILABLE_STATUS = "anonymization_unavailable"
2161
+ _ANON_UNAVAILABLE_REMEDY = "cctally cache-sync --source codex --rebuild"
2162
+
2163
+
2164
+ def _anon_unavailable_body(undecodable_cwd_rows: int) -> dict:
2165
+ return {
2166
+ "status": _ANON_UNAVAILABLE_STATUS,
2167
+ "undecodable_cwd_rows": int(undecodable_cwd_rows),
2168
+ "remedy": _ANON_UNAVAILABLE_REMEDY,
2169
+ }
2170
+
2171
+
2172
+ def _anon_ambiguous_body(ambiguous_cwd_rows: int) -> dict:
2173
+ """Typed refusal for a scoped path whose account ownership is unproven."""
2174
+ return {
2175
+ "status": _ANON_UNAVAILABLE_STATUS,
2176
+ "ambiguous_cwd_rows": int(ambiguous_cwd_rows),
2177
+ "reason": "ambiguous_account_provenance",
2178
+ "remedy": "use a raw export or omit the account scope",
2179
+ }
2180
+
2181
+
1752
2182
  def _handle_get_conversation_export_impl(handler, path: str) -> None:
1753
2183
  """``GET /api/conversation/<sid>/export?scope=<all|prompts|chat|recipe>``
1754
2184
  — whole-session Markdown (issue #217 S5 F1/F5).
@@ -1806,15 +2236,23 @@ def _handle_get_conversation_export_impl(handler, path: str) -> None:
1806
2236
  cq = handler._conversation_query()
1807
2237
  anon = sys.modules["cctally"]._load_sibling("_lib_conversation_anon")
1808
2238
  srcs = {cref.source} if cref else set()
1809
- plan = cq.build_anon_plan_for_sources(
2239
+ result = cq.build_anon_plan_for_sources(
1810
2240
  conn, home_dir=_os.path.expanduser("~"), sources=srcs)
1811
- return {**env, "markdown": anon.scrub_text(env["markdown"], plan)}
2241
+ if result.undecodable_cwd_rows:
2242
+ return _anon_unavailable_body(result.undecodable_cwd_rows)
2243
+ if result.ambiguous_cwd_rows:
2244
+ return _anon_ambiguous_body(result.ambiguous_cwd_rows)
2245
+ return {**env,
2246
+ "markdown": anon.scrub_text(env["markdown"], result.plan)}
1812
2247
 
1813
2248
  ok, env = handler._run_conversation_query(
1814
2249
  _q_kernel, "/api/conversation/export")
1815
2250
  if not ok:
1816
2251
  return
1817
2252
  status = (env or {}).get("status")
2253
+ if status == _ANON_UNAVAILABLE_STATUS:
2254
+ handler._respond_json(409, env)
2255
+ return
1818
2256
  if status == "ok":
1819
2257
  data = env["markdown"].encode("utf-8")
1820
2258
  handler.send_response(200)
@@ -1896,14 +2334,22 @@ def _handle_get_conversation_anon_map_impl(handler, path: str) -> None:
1896
2334
  exists = cq.conversation_exists(conn, cref.native_key)
1897
2335
  if not exists:
1898
2336
  return None
1899
- plan = cq.build_anon_plan_for_sources(
2337
+ result = cq.build_anon_plan_for_sources(
1900
2338
  conn, home_dir=_os.path.expanduser("~"), sources={cref.source})
1901
- return anon.plan_to_wire(plan)
2339
+ if result.undecodable_cwd_rows:
2340
+ return _anon_unavailable_body(result.undecodable_cwd_rows)
2341
+ if result.ambiguous_cwd_rows:
2342
+ return _anon_ambiguous_body(result.ambiguous_cwd_rows)
2343
+ return anon.plan_to_wire(result.plan)
1902
2344
 
1903
2345
  ok, body = handler._run_conversation_query(
1904
2346
  _q_kernel, "/api/conversation/anon-map")
1905
2347
  if not ok:
1906
2348
  return
2349
+ if isinstance(body, dict) and body.get(
2350
+ "status") == _ANON_UNAVAILABLE_STATUS:
2351
+ handler._respond_json(409, body)
2352
+ return
1907
2353
  if body is None:
1908
2354
  handler._respond_json(
1909
2355
  404, {"status": "not_found", "conversation_key": session_id})