cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -47,6 +47,7 @@ import collections
|
|
|
47
47
|
import dataclasses
|
|
48
48
|
import hashlib
|
|
49
49
|
import secrets
|
|
50
|
+
import select
|
|
50
51
|
import threading
|
|
51
52
|
import time
|
|
52
53
|
|
|
@@ -115,10 +116,16 @@ _OUTLINE_TRANSFER_ITEM_CAP = 32 * 1024 * 1024
|
|
|
115
116
|
_OUTLINE_TRANSFER_TOTAL_CAP = 64 * 1024 * 1024
|
|
116
117
|
_OUTLINE_TRANSFER_COUNT_CAP = 128
|
|
117
118
|
_OUTLINE_TRANSFER_TTL = 120.0
|
|
119
|
+
_OUTLINE_TRANSFER_REQUEST_DEADLINE = 15.0
|
|
120
|
+
_OUTLINE_TRANSFER_MAX_WAITERS = 32
|
|
121
|
+
_OUTLINE_TRANSFER_MAX_WAITERS_PER_TOKEN = 16
|
|
122
|
+
_OUTLINE_TRANSFER_MAX_PRODUCERS = 2
|
|
118
123
|
_OUTLINE_TRANSFERS = collections.OrderedDict()
|
|
119
124
|
_OUTLINE_TRANSFERS_BYTES = 0
|
|
120
125
|
_OUTLINE_TRANSFERS_EVICTIONS = 0
|
|
121
126
|
_OUTLINE_TRANSFERS_FALLBACKS = 0
|
|
127
|
+
_OUTLINE_TRANSFERS_WAITERS = 0
|
|
128
|
+
_OUTLINE_TRANSFERS_PRODUCERS = 0
|
|
122
129
|
_OUTLINE_TRANSFERS_LOCK = threading.Lock()
|
|
123
130
|
|
|
124
131
|
|
|
@@ -129,6 +136,10 @@ class _OutlineTransferRecord:
|
|
|
129
136
|
payload: bytes | None = None
|
|
130
137
|
digest: str | None = None
|
|
131
138
|
state: str = "pending"
|
|
139
|
+
waiters: int = 0
|
|
140
|
+
deadline: float | None = None
|
|
141
|
+
failure_status: int | None = None
|
|
142
|
+
failure_body: dict | None = None
|
|
132
143
|
cancelled: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
133
144
|
ready: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
134
145
|
|
|
@@ -148,6 +159,8 @@ def outline_transfer_cache_stats():
|
|
|
148
159
|
"maxEntries": _OUTLINE_TRANSFER_COUNT_CAP,
|
|
149
160
|
"evictionCount": int(_OUTLINE_TRANSFERS_EVICTIONS),
|
|
150
161
|
"fallbackCount": int(_OUTLINE_TRANSFERS_FALLBACKS),
|
|
162
|
+
"admittedWaiters": int(_OUTLINE_TRANSFERS_WAITERS),
|
|
163
|
+
"activeProducers": int(_OUTLINE_TRANSFERS_PRODUCERS),
|
|
151
164
|
}
|
|
152
165
|
|
|
153
166
|
|
|
@@ -191,8 +204,9 @@ def _store_outline_transfer(builder):
|
|
|
191
204
|
return {"token": token, "chunk_size": _OUTLINE_TRANSFER_CHUNK}
|
|
192
205
|
|
|
193
206
|
|
|
194
|
-
def _claim_outline_transfer(token):
|
|
207
|
+
def _claim_outline_transfer(token, *, admit=False):
|
|
195
208
|
"""Atomically claim a pending builder or join its in-flight generation."""
|
|
209
|
+
global _OUTLINE_TRANSFERS_WAITERS, _OUTLINE_TRANSFERS_PRODUCERS
|
|
196
210
|
now = time.monotonic()
|
|
197
211
|
with _OUTLINE_TRANSFERS_LOCK:
|
|
198
212
|
_prune_outline_transfers_locked(now)
|
|
@@ -202,12 +216,49 @@ def _claim_outline_transfer(token):
|
|
|
202
216
|
_OUTLINE_TRANSFERS.move_to_end(token)
|
|
203
217
|
if record.payload is not None:
|
|
204
218
|
return "ready", record, None
|
|
219
|
+
if admit:
|
|
220
|
+
if (_OUTLINE_TRANSFERS_WAITERS >= _OUTLINE_TRANSFER_MAX_WAITERS
|
|
221
|
+
or record.waiters >= _OUTLINE_TRANSFER_MAX_WAITERS_PER_TOKEN
|
|
222
|
+
or (record.state == "pending" and
|
|
223
|
+
_OUTLINE_TRANSFERS_PRODUCERS >=
|
|
224
|
+
_OUTLINE_TRANSFER_MAX_PRODUCERS)):
|
|
225
|
+
return "busy", record, None
|
|
226
|
+
record.waiters += 1
|
|
227
|
+
_OUTLINE_TRANSFERS_WAITERS += 1
|
|
205
228
|
if record.state == "pending":
|
|
206
229
|
record.state = "building"
|
|
230
|
+
if admit:
|
|
231
|
+
record.deadline = min(
|
|
232
|
+
now + _OUTLINE_TRANSFER_REQUEST_DEADLINE,
|
|
233
|
+
record.created + _OUTLINE_TRANSFER_TTL,
|
|
234
|
+
)
|
|
235
|
+
_OUTLINE_TRANSFERS_PRODUCERS += 1
|
|
207
236
|
return "build", record, record.builder
|
|
208
237
|
return "wait", record, None
|
|
209
238
|
|
|
210
239
|
|
|
240
|
+
def _release_outline_transfer_waiter(token, record):
|
|
241
|
+
"""An abandoned last reader cancels its producer without releasing its slot."""
|
|
242
|
+
global _OUTLINE_TRANSFERS_WAITERS
|
|
243
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
244
|
+
record.waiters -= 1
|
|
245
|
+
_OUTLINE_TRANSFERS_WAITERS -= 1
|
|
246
|
+
if (record.waiters == 0 and record.state == "building"
|
|
247
|
+
and _OUTLINE_TRANSFERS.get(token) is record):
|
|
248
|
+
_drop_outline_transfer_locked(token)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _finish_outline_transfer_failure(token, record, status, body):
|
|
252
|
+
"""Publish one failure to admitted readers and retire only this generation."""
|
|
253
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
254
|
+
if _OUTLINE_TRANSFERS.get(token) is not record:
|
|
255
|
+
return False
|
|
256
|
+
record.failure_status = status
|
|
257
|
+
record.failure_body = body
|
|
258
|
+
_drop_outline_transfer_locked(token)
|
|
259
|
+
return True
|
|
260
|
+
|
|
261
|
+
|
|
211
262
|
def _outline_transfer_builder(token):
|
|
212
263
|
"""Compatibility probe: claim one builder, ``False`` if joined/ready."""
|
|
213
264
|
claim = _claim_outline_transfer(token)
|
|
@@ -225,6 +276,9 @@ def _materialize_outline_transfer(token, payload, *, claim=None):
|
|
|
225
276
|
with _OUTLINE_TRANSFERS_LOCK:
|
|
226
277
|
current = _OUTLINE_TRANSFERS.get(token)
|
|
227
278
|
if current is not None and (claim is None or current is claim):
|
|
279
|
+
current.failure_status = 413
|
|
280
|
+
current.failure_body = {
|
|
281
|
+
"error": "outline exceeds progressive transfer limit"}
|
|
228
282
|
_drop_outline_transfer_locked(token)
|
|
229
283
|
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
230
284
|
return "too_large"
|
|
@@ -266,6 +320,9 @@ def _materialize_outline_transfer(token, payload, *, claim=None):
|
|
|
266
320
|
None,
|
|
267
321
|
)
|
|
268
322
|
if victim is None:
|
|
323
|
+
record.failure_status = 413
|
|
324
|
+
record.failure_body = {
|
|
325
|
+
"error": "outline exceeds progressive transfer limit"}
|
|
269
326
|
_drop_outline_transfer_locked(token)
|
|
270
327
|
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
271
328
|
return "too_large"
|
|
@@ -292,6 +349,9 @@ def _reject_outline_transfer_too_large(token, *, claim) -> bool:
|
|
|
292
349
|
current = _OUTLINE_TRANSFERS.get(token)
|
|
293
350
|
if current is None or current is not claim:
|
|
294
351
|
return False
|
|
352
|
+
current.failure_status = 413
|
|
353
|
+
current.failure_body = {
|
|
354
|
+
"error": "outline exceeds progressive transfer limit"}
|
|
295
355
|
_drop_outline_transfer_locked(token)
|
|
296
356
|
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
297
357
|
return True
|
|
@@ -655,8 +715,13 @@ def _handle_qualified_facets(handler, qs_raw, source):
|
|
|
655
715
|
if not ok:
|
|
656
716
|
return
|
|
657
717
|
facets = (body or {}).get("facets") or {"projects": [], "models": []}
|
|
658
|
-
|
|
659
|
-
|
|
718
|
+
# #717: the SECOND reshaping site. This rebuilds the response from two
|
|
719
|
+
# named keys, so an unknown key added upstream reaches no client unless it
|
|
720
|
+
# is carried here too.
|
|
721
|
+
payload = {"status": (body or {}).get("status"), "facets": facets}
|
|
722
|
+
if (body or {}).get("filter_degraded"):
|
|
723
|
+
payload["filter_degraded"] = True
|
|
724
|
+
handler._respond_json(200, payload)
|
|
660
725
|
|
|
661
726
|
|
|
662
727
|
def _handle_qualified_search(handler, qs_raw, source):
|
|
@@ -712,6 +777,156 @@ def _parse_search_kind_impl(handler, q, valid=_CONV_SEARCH_KINDS):
|
|
|
712
777
|
return None
|
|
713
778
|
return kind
|
|
714
779
|
|
|
780
|
+
#: #780 — how many times a read route re-attempts admission before it degrades.
|
|
781
|
+
#: Bounded by count, never by wall clock: the opener's own busy timeout is
|
|
782
|
+
#: already route-bounded, and an unbounded retry against a rebuild that holds
|
|
783
|
+
#: the store for minutes would pin a request thread for the whole rebuild.
|
|
784
|
+
_CONVERSATION_READ_ATTEMPTS = 3
|
|
785
|
+
_CONVERSATION_READ_RETRY_SLEEP_S = 0.02
|
|
786
|
+
|
|
787
|
+
#: The empty-but-valid body each named read route serves alongside the typed
|
|
788
|
+
#: degraded marker. Keyed on the caller's own log label, so no call site
|
|
789
|
+
#: changes and a route with no entry still gets the marker.
|
|
790
|
+
_CONVERSATION_DEGRADED_SHAPES = {
|
|
791
|
+
"/api/conversations": {"conversations": [], "total": 0},
|
|
792
|
+
"/api/conversations/facets": {"projects": [], "models": []},
|
|
793
|
+
"/api/conversation/search": {"results": [], "total": 0},
|
|
794
|
+
"/api/conversation/find": {"matches": [], "total": 0},
|
|
795
|
+
"/api/conversation/prompts": {"prompts": [], "total": 0},
|
|
796
|
+
"/api/conversation": {},
|
|
797
|
+
"/api/conversation/outline": {},
|
|
798
|
+
"/api/conversation/payload": {},
|
|
799
|
+
"/api/conversation/media": {},
|
|
800
|
+
"/api/conversation/export": {},
|
|
801
|
+
# The live-tail preflight answers before any SSE byte is committed, so its
|
|
802
|
+
# refusal is plain JSON like every other route's. An empty shape is right
|
|
803
|
+
# here: the client's own backstop tick is what recovers, and there is no
|
|
804
|
+
# partial stream to render.
|
|
805
|
+
"/api/conversation/events": {},
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def _reader_unavailable():
|
|
810
|
+
"""The typed reader-admission condition, resolved lazily.
|
|
811
|
+
|
|
812
|
+
`_cctally_dashboard` re-exports the cache module's opener, and the tests
|
|
813
|
+
monkeypatch that re-export, so the class is read through the same module
|
|
814
|
+
object the opener came from rather than imported at module scope.
|
|
815
|
+
"""
|
|
816
|
+
return sys.modules["_cctally_dashboard"].ConversationReaderUnavailable
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _conversation_degraded_body(log_label, reason):
|
|
820
|
+
"""One typed degraded envelope for a read route that could not be admitted.
|
|
821
|
+
|
|
822
|
+
HTTP 200, because the condition is transient and expected: maintenance is
|
|
823
|
+
running, or a writer has not yet advanced the schema. It carries the
|
|
824
|
+
route's own empty shape so a client that ignores the marker renders an
|
|
825
|
+
empty surface rather than failing to parse, and it names no SQLite text, no
|
|
826
|
+
SQL and no filesystem path — the privacy posture the 500 envelope already
|
|
827
|
+
held.
|
|
828
|
+
"""
|
|
829
|
+
body = dict(_CONVERSATION_DEGRADED_SHAPES.get(log_label, {}))
|
|
830
|
+
body["status"] = "degraded"
|
|
831
|
+
body["degraded_reason"] = reason
|
|
832
|
+
return body
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def _full_open_or_refuse():
|
|
836
|
+
"""Open write-capable, and refuse if the legacy bridge is STILL owed (#802).
|
|
837
|
+
|
|
838
|
+
The full opener is what fixes an interrupted migration 028: it runs
|
|
839
|
+
`_import_legacy_conversation_rows`, which moves the transcript rows out of
|
|
840
|
+
`cache.db` and derives the browse rollup from them. Under `dashboard
|
|
841
|
+
--no-sync` the process policy suppresses that derivation, so this opener
|
|
842
|
+
returns a store where the rows are still in `cache.db` and the rollup is
|
|
843
|
+
still empty — and `list_conversation_facets` reads the rollup's
|
|
844
|
+
`project_label` with no authoritative gate, so the conversation list and
|
|
845
|
+
the browse project filter would both render empty for the life of the
|
|
846
|
+
process, with no pending state and nothing said.
|
|
847
|
+
|
|
848
|
+
Spec §9 permits exactly this alternative for a route that cannot inherit
|
|
849
|
+
the derivation policy: return a typed degraded response instead. The
|
|
850
|
+
outcome is checked rather than the policy, so a bridge that COMPLETED is
|
|
851
|
+
byte-unchanged, and a future third reason the bridge could not run is
|
|
852
|
+
covered without a second predicate.
|
|
853
|
+
|
|
854
|
+
The claim is about the outcome, not about the mode. It was first written as
|
|
855
|
+
"every mode allowed to do work is byte-unchanged", and that is too strong:
|
|
856
|
+
`_import_legacy_conversation_rows` is fail-soft per table (`except
|
|
857
|
+
sqlite3.Error: continue`), so a mode that IS allowed to do work but whose
|
|
858
|
+
import raised on one table leaves the predicate true and is refused here,
|
|
859
|
+
where the old code returned the connection. That refusal is the better
|
|
860
|
+
answer — the old code served an empty conversation surface over rows it had
|
|
861
|
+
just failed to move, and said nothing about it — but it is a change, so it
|
|
862
|
+
is stated rather than covered by an absolute.
|
|
863
|
+
"""
|
|
864
|
+
dash = sys.modules["_cctally_dashboard"]
|
|
865
|
+
cache_mod = sys.modules["_cctally_cache"]
|
|
866
|
+
conn = dash.open_conversations_db()
|
|
867
|
+
if not cache_mod.conversation_legacy_bridge_pending(conn):
|
|
868
|
+
return conn
|
|
869
|
+
conn.close()
|
|
870
|
+
raise dash.LegacyBridgePending(
|
|
871
|
+
"the legacy transcript bridge is owed and derivations are suppressed")
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _open_conversation_reader(log_label):
|
|
875
|
+
"""Admit one read route, with a bounded retry (#780).
|
|
876
|
+
|
|
877
|
+
Reads through the read-only opener, which performs no schema apply, no
|
|
878
|
+
migration dispatch, no PRAGMA write and no chmod, so a browse is no longer
|
|
879
|
+
a writer that can lose the SQLite write lock to a reclaim pass.
|
|
880
|
+
|
|
881
|
+
A store that is simply not present yet falls back to the full opener, which
|
|
882
|
+
creates and migrates it. That is the first-run path, not the steady state,
|
|
883
|
+
and removing it would make an empty installation serve a degraded envelope
|
|
884
|
+
forever with no process willing to create the store.
|
|
885
|
+
|
|
886
|
+
BOTH fallbacks probe the maintenance flock first. The full opener's
|
|
887
|
+
admission is a plain `LOCK_SH` with no timeout, so falling back to it
|
|
888
|
+
during a rebuild queued the request behind the whole rebuild — 716.9 s in
|
|
889
|
+
the measured run — which is the failure this opener exists to remove,
|
|
890
|
+
reintroduced on the two paths that leave it. The probe refuses instead, and
|
|
891
|
+
the route serves the same typed degraded envelope every other refusal
|
|
892
|
+
already produces. The legacy-bridge fallback matters more than the
|
|
893
|
+
first-run one here: `conversation_legacy_bridge_pending` is durable state
|
|
894
|
+
after an interrupted migration 028, so it is not a one-off startup case.
|
|
895
|
+
|
|
896
|
+
Both fallbacks go through `_full_open_or_refuse`, which serves the opened
|
|
897
|
+
connection only when the bridge is no longer owed. Under `--no-sync` the
|
|
898
|
+
bridge is suppressed and cannot clear, so the route answers the typed
|
|
899
|
+
degraded envelope rather than an empty conversation surface (#802).
|
|
900
|
+
"""
|
|
901
|
+
dash = sys.modules["_cctally_dashboard"]
|
|
902
|
+
cache_mod = sys.modules["_cctally_cache"]
|
|
903
|
+
last = None
|
|
904
|
+
for attempt in range(_CONVERSATION_READ_ATTEMPTS):
|
|
905
|
+
try:
|
|
906
|
+
conn = dash.open_conversations_db_readonly()
|
|
907
|
+
except dash.MaintenanceInProgress as exc:
|
|
908
|
+
last = exc
|
|
909
|
+
if attempt + 1 < _CONVERSATION_READ_ATTEMPTS:
|
|
910
|
+
time.sleep(_CONVERSATION_READ_RETRY_SLEEP_S)
|
|
911
|
+
continue
|
|
912
|
+
raise
|
|
913
|
+
except dash.ConversationReaderUnavailable as exc:
|
|
914
|
+
if type(exc) is dash.ConversationReaderUnavailable:
|
|
915
|
+
# The store (or the accounting store it attaches) is absent.
|
|
916
|
+
dash.probe_conversations_maintenance_free()
|
|
917
|
+
return _full_open_or_refuse()
|
|
918
|
+
raise
|
|
919
|
+
if cache_mod.conversation_legacy_bridge_pending(conn):
|
|
920
|
+
# An interrupted migration 028 left transcript rows in cache.db.
|
|
921
|
+
# The bridge that moves them is a writer, so hand the open back to
|
|
922
|
+
# the full opener rather than serving an empty surface over rows
|
|
923
|
+
# that are present but not yet in this store.
|
|
924
|
+
conn.close()
|
|
925
|
+
dash.probe_conversations_maintenance_free()
|
|
926
|
+
return _full_open_or_refuse()
|
|
927
|
+
return conn
|
|
928
|
+
raise last # unreachable: the loop either returns or raises
|
|
929
|
+
|
|
715
930
|
def _run_conversation_query_impl(
|
|
716
931
|
handler, kernel_call, log_label, *, cancelled=None,
|
|
717
932
|
):
|
|
@@ -728,7 +943,17 @@ def _run_conversation_query_impl(
|
|
|
728
943
|
``{type}: {msg}`` 500.
|
|
729
944
|
"""
|
|
730
945
|
try:
|
|
731
|
-
conn =
|
|
946
|
+
conn = _open_conversation_reader(log_label)
|
|
947
|
+
except _reader_unavailable() as exc:
|
|
948
|
+
# #780: maintenance and schema admission are NOT server errors. A
|
|
949
|
+
# rebuild, a reclaim pass or a store one migration behind used to reach
|
|
950
|
+
# the reader as a 500, which is exactly the 5xx this session exists to
|
|
951
|
+
# remove. Answer with the route's own typed degraded envelope instead.
|
|
952
|
+
handler.log_error(
|
|
953
|
+
"%s transcript store degraded: %s", log_label, exc.reason)
|
|
954
|
+
handler._respond_json(200, _conversation_degraded_body(
|
|
955
|
+
log_label, exc.reason))
|
|
956
|
+
return False, None
|
|
732
957
|
except (sqlite3.DatabaseError, OSError) as exc:
|
|
733
958
|
handler.log_error("%s transcript store open failed: %r", log_label, exc)
|
|
734
959
|
handler._respond_json(
|
|
@@ -1021,7 +1246,9 @@ def _run_conversation_events_stream(
|
|
|
1021
1246
|
if passive:
|
|
1022
1247
|
# Frozen-data contract: no ingest, no emit. Keep-alive only.
|
|
1023
1248
|
while True:
|
|
1024
|
-
|
|
1249
|
+
if _request_peer_gone(
|
|
1250
|
+
handler, timeout=_LIVE_TAIL_KEEPALIVE):
|
|
1251
|
+
return
|
|
1025
1252
|
handler.wfile.write(b": keep-alive\n\n")
|
|
1026
1253
|
handler.wfile.flush()
|
|
1027
1254
|
|
|
@@ -1056,7 +1283,8 @@ def _run_conversation_events_stream(
|
|
|
1056
1283
|
idle = 0.0
|
|
1057
1284
|
cycles = 0
|
|
1058
1285
|
while True:
|
|
1059
|
-
|
|
1286
|
+
if _request_peer_gone(handler, timeout=_LIVE_TAIL_POLL_INTERVAL):
|
|
1287
|
+
return
|
|
1060
1288
|
cycles += 1
|
|
1061
1289
|
changed = watch.changed_paths(files, seen)
|
|
1062
1290
|
if changed:
|
|
@@ -1103,6 +1331,62 @@ def _run_conversation_events_stream(
|
|
|
1103
1331
|
idle = 0.0
|
|
1104
1332
|
|
|
1105
1333
|
|
|
1334
|
+
def _live_tail_read_connection():
|
|
1335
|
+
"""The READ connection a live-tail stream holds for its whole lifetime.
|
|
1336
|
+
|
|
1337
|
+
Deliberately the full opener, not `open_conversations_db_readonly`, and the
|
|
1338
|
+
reason is measured rather than assumed. Moving it to the read-only opener
|
|
1339
|
+
made `test_codex_child_discovery_emits_tail` stop emitting a tail: the
|
|
1340
|
+
stream reached `ready` and `baselined`, the child rollout was ingested
|
|
1341
|
+
through the separate writer below, and the long-lived read connection never
|
|
1342
|
+
observed the committed growth, so `seen` never changed and no `event: tail`
|
|
1343
|
+
was ever pushed.
|
|
1344
|
+
|
|
1345
|
+
That is a different problem from the one #780 is about. A read ROUTE opens
|
|
1346
|
+
per request, so its opener's write-capable pragmas are what compete with
|
|
1347
|
+
maintenance for the SQLite write lock on every browse; a live-tail stream
|
|
1348
|
+
opens ONCE and then holds the connection for minutes, so it contributes one
|
|
1349
|
+
open, not one per request. Task 3.3's requirement — that every live-tail
|
|
1350
|
+
INGEST branch takes its own write connection — is met by `_live_tail_write`
|
|
1351
|
+
below, and that is the part that would otherwise make each stream a writer.
|
|
1352
|
+
|
|
1353
|
+
What did NOT survive is entering that opener unconditionally. Its
|
|
1354
|
+
maintenance admission is a blocking `LOCK_SH`, so a stream opened during a
|
|
1355
|
+
rebuild waited out the rebuild — 716.9 s measured — holding a server thread
|
|
1356
|
+
the whole time and answering nothing. §4a names the live-tail preflight
|
|
1357
|
+
among the routes that must fail soft, so the flock is probed non-blocking
|
|
1358
|
+
first and the two callers turn `MaintenanceInProgress` into the same typed
|
|
1359
|
+
degraded envelope every other read route serves.
|
|
1360
|
+
|
|
1361
|
+
It goes through `_full_open_or_refuse` for the same reason the read routes
|
|
1362
|
+
do: under `--no-sync` the legacy bridge is suppressed, so a stream opened
|
|
1363
|
+
over an owed bridge would tail a store whose transcript rows are still in
|
|
1364
|
+
`cache.db` and report nothing for as long as it ran (#802).
|
|
1365
|
+
"""
|
|
1366
|
+
dash = sys.modules["_cctally_dashboard"]
|
|
1367
|
+
dash.probe_conversations_maintenance_free()
|
|
1368
|
+
return _full_open_or_refuse()
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
def _live_tail_write(work):
|
|
1372
|
+
"""Run one live-tail ingest on its OWN write connection (#780).
|
|
1373
|
+
|
|
1374
|
+
Live-tail is the single write exception among the conversation routes.
|
|
1375
|
+
Its own read connection is NOT read-only — `_live_tail_read_connection`
|
|
1376
|
+
above keeps the full opener, and says why — but that connection must still
|
|
1377
|
+
not be the one that ingests: a stream holds it for minutes, and driving a
|
|
1378
|
+
sync on it mixes the stream's long-lived read transaction with writes. The
|
|
1379
|
+
ingest branches therefore open a writer, use it, and close it. The stream's
|
|
1380
|
+
read connection is refreshed by the caller's next query, which starts a
|
|
1381
|
+
fresh read transaction and so observes what this writer committed.
|
|
1382
|
+
"""
|
|
1383
|
+
writer = sys.modules["_cctally_dashboard"].open_conversations_db()
|
|
1384
|
+
try:
|
|
1385
|
+
return work(writer)
|
|
1386
|
+
finally:
|
|
1387
|
+
writer.close()
|
|
1388
|
+
|
|
1389
|
+
|
|
1106
1390
|
def _bare_conversation_events(
|
|
1107
1391
|
handler, session_id: str, account_key: str | None = None,
|
|
1108
1392
|
) -> None:
|
|
@@ -1111,7 +1395,16 @@ def _bare_conversation_events(
|
|
|
1111
1395
|
cq = handler._conversation_query()
|
|
1112
1396
|
passive = bool(type(handler).no_sync)
|
|
1113
1397
|
try:
|
|
1114
|
-
conn =
|
|
1398
|
+
conn = _live_tail_read_connection()
|
|
1399
|
+
except _reader_unavailable() as exc:
|
|
1400
|
+
# #780: maintenance is not a server error and not a passive stream
|
|
1401
|
+
# either. Answer the typed degraded envelope BEFORE any SSE byte, the
|
|
1402
|
+
# same way every other read route does.
|
|
1403
|
+
handler._respond_json(
|
|
1404
|
+
200, _conversation_degraded_body(
|
|
1405
|
+
"/api/conversation/events", getattr(exc, "reason",
|
|
1406
|
+
"unavailable")))
|
|
1407
|
+
return
|
|
1115
1408
|
except (sqlite3.DatabaseError, OSError):
|
|
1116
1409
|
if account_key is not None:
|
|
1117
1410
|
# A qualified account boundary must be established before any SSE
|
|
@@ -1139,7 +1432,14 @@ def _bare_conversation_events(
|
|
|
1139
1432
|
def _ingest(changed):
|
|
1140
1433
|
_advance_live_tail_accounting(handler, changed, sync_cache, "claude")
|
|
1141
1434
|
if account_key is None:
|
|
1142
|
-
|
|
1435
|
+
# #780: ingest takes its OWN write connection. The read connection
|
|
1436
|
+
# this stream then queries is opened read-only, so writing through
|
|
1437
|
+
# it would fail outright — and even on a writable connection,
|
|
1438
|
+
# driving ingestion from the connection a long-lived SSE stream
|
|
1439
|
+
# reads from is what made every live-tail a writer.
|
|
1440
|
+
return _live_tail_write(
|
|
1441
|
+
lambda writer: sync_claude_conversations(
|
|
1442
|
+
writer, only_paths=set(changed)))
|
|
1143
1443
|
before = conn.execute(
|
|
1144
1444
|
"SELECT COUNT(*),MAX(id) FROM conversation_messages "
|
|
1145
1445
|
"WHERE session_id=?",
|
|
@@ -1250,8 +1550,22 @@ def _qualified_conversation_events(
|
|
|
1250
1550
|
targeted ingest + the directory-frontier child discovery."""
|
|
1251
1551
|
disp = _conversation_dispatch_impl()
|
|
1252
1552
|
try:
|
|
1253
|
-
conn =
|
|
1553
|
+
conn = _live_tail_read_connection()
|
|
1554
|
+
except _reader_unavailable() as exc:
|
|
1555
|
+
# #780: maintenance holds the store. The preflight has committed no SSE
|
|
1556
|
+
# bytes yet, so this is answerable as the same typed degraded envelope
|
|
1557
|
+
# every other read route serves — and it must be, because a passive
|
|
1558
|
+
# stream here would look to the client like a conversation with no
|
|
1559
|
+
# turns rather than a store that is busy.
|
|
1560
|
+
handler._respond_json(
|
|
1561
|
+
200, _conversation_degraded_body(
|
|
1562
|
+
"/api/conversation/events", getattr(exc, "reason",
|
|
1563
|
+
"unavailable")))
|
|
1564
|
+
return
|
|
1254
1565
|
except (sqlite3.DatabaseError, OSError):
|
|
1566
|
+
# Any other open failure takes the passive keep-alive stream: a
|
|
1567
|
+
# genuinely unopenable store — an incomplete recovery, a live repair
|
|
1568
|
+
# marker, an I/O error.
|
|
1255
1569
|
conn = None
|
|
1256
1570
|
|
|
1257
1571
|
if conn is None:
|
|
@@ -1310,7 +1624,9 @@ def _qualified_conversation_events(
|
|
|
1310
1624
|
if account_key is None:
|
|
1311
1625
|
_advance_live_tail_accounting(
|
|
1312
1626
|
handler, changed, sync_codex_cache, "codex")
|
|
1313
|
-
return
|
|
1627
|
+
return _live_tail_write(
|
|
1628
|
+
lambda writer: sync_codex_conversations(
|
|
1629
|
+
writer, only_paths=set(changed)))
|
|
1314
1630
|
before = conn.execute(
|
|
1315
1631
|
"SELECT COUNT(*),MAX(id) FROM codex_conversation_messages "
|
|
1316
1632
|
"WHERE conversation_key=?",
|
|
@@ -1356,7 +1672,9 @@ def _qualified_conversation_events(
|
|
|
1356
1672
|
_advance_live_tail_accounting(
|
|
1357
1673
|
handler, changed, sync_cache, "claude")
|
|
1358
1674
|
if account_key is None:
|
|
1359
|
-
return
|
|
1675
|
+
return _live_tail_write(
|
|
1676
|
+
lambda writer: sync_claude_conversations(
|
|
1677
|
+
writer, only_paths=set(changed)))
|
|
1360
1678
|
before = conn.execute(
|
|
1361
1679
|
"SELECT COUNT(*),MAX(id) FROM conversation_messages "
|
|
1362
1680
|
"WHERE session_id=?",
|
|
@@ -1538,7 +1856,8 @@ def _handle_get_conversation_payload_impl(handler, path: str) -> None:
|
|
|
1538
1856
|
|
|
1539
1857
|
def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
1540
1858
|
"""``GET /api/conversation/<sid>/outline`` — full-session skeleton +
|
|
1541
|
-
session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404
|
|
1859
|
+
session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404
|
|
1860
|
+
for the legacy form, typed absence on the progressive preflight.
|
|
1542
1861
|
"""
|
|
1543
1862
|
if not handler._require_transcripts_allowed():
|
|
1544
1863
|
return
|
|
@@ -1604,7 +1923,7 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
|
1604
1923
|
if not ok:
|
|
1605
1924
|
return
|
|
1606
1925
|
if not exists:
|
|
1607
|
-
handler.
|
|
1926
|
+
handler._respond_json(200, {"status": "not_found"})
|
|
1608
1927
|
return
|
|
1609
1928
|
|
|
1610
1929
|
def _build_bare(transfer_handler, cancelled):
|
|
@@ -1637,6 +1956,99 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
|
1637
1956
|
handler._respond_json(200, body)
|
|
1638
1957
|
|
|
1639
1958
|
|
|
1959
|
+
class _OutlineTransferBuildHandler:
|
|
1960
|
+
"""Give a producer read access to its request without writing its socket."""
|
|
1961
|
+
|
|
1962
|
+
def __init__(self, handler):
|
|
1963
|
+
self._handler = handler
|
|
1964
|
+
self.response = None
|
|
1965
|
+
|
|
1966
|
+
def __getattr__(self, name):
|
|
1967
|
+
return getattr(self._handler, name)
|
|
1968
|
+
|
|
1969
|
+
def _respond_json(self, status, body):
|
|
1970
|
+
self.response = (status, body)
|
|
1971
|
+
|
|
1972
|
+
|
|
1973
|
+
def _produce_outline_transfer(token, record, builder, handler):
|
|
1974
|
+
"""Build off the request thread; the producer slot lasts until real exit."""
|
|
1975
|
+
global _OUTLINE_TRANSFERS_PRODUCERS
|
|
1976
|
+
proxy = _OutlineTransferBuildHandler(handler)
|
|
1977
|
+
try:
|
|
1978
|
+
try:
|
|
1979
|
+
ok, full_body = builder(proxy, record.cancelled.is_set)
|
|
1980
|
+
if record.cancelled.is_set():
|
|
1981
|
+
return
|
|
1982
|
+
if ok is None or full_body is None and ok:
|
|
1983
|
+
_finish_outline_transfer_failure(
|
|
1984
|
+
token, record, 410,
|
|
1985
|
+
{"error": "outline transfer expired"})
|
|
1986
|
+
return
|
|
1987
|
+
if not ok:
|
|
1988
|
+
status, body = proxy.response or (
|
|
1989
|
+
500, {"error": "outline transfer failed"})
|
|
1990
|
+
_finish_outline_transfer_failure(token, record, status, body)
|
|
1991
|
+
return
|
|
1992
|
+
payload = encode_dashboard_json_bytes_capped(
|
|
1993
|
+
full_body, max_bytes=_OUTLINE_TRANSFER_ITEM_CAP)
|
|
1994
|
+
if payload is None:
|
|
1995
|
+
_reject_outline_transfer_too_large(token, claim=record)
|
|
1996
|
+
else:
|
|
1997
|
+
_materialize_outline_transfer(token, payload, claim=record)
|
|
1998
|
+
except Exception as exc: # noqa: BLE001
|
|
1999
|
+
handler.log_error("outline transfer build failed: %r", exc)
|
|
2000
|
+
_finish_outline_transfer_failure(
|
|
2001
|
+
token, record, 500, {"error": "outline transfer failed"})
|
|
2002
|
+
finally:
|
|
2003
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
2004
|
+
_OUTLINE_TRANSFERS_PRODUCERS -= 1
|
|
2005
|
+
|
|
2006
|
+
|
|
2007
|
+
def _request_peer_gone(handler, *, timeout=0.0):
|
|
2008
|
+
"""Detect a closed request socket while no response is being written."""
|
|
2009
|
+
conn = getattr(handler, "connection", None)
|
|
2010
|
+
if conn is None:
|
|
2011
|
+
return False
|
|
2012
|
+
try:
|
|
2013
|
+
if not select.select([conn], [], [], timeout)[0]:
|
|
2014
|
+
return False
|
|
2015
|
+
return not conn.recv(
|
|
2016
|
+
1, socket.MSG_PEEK | getattr(socket, "MSG_DONTWAIT", 0))
|
|
2017
|
+
except (OSError, ValueError):
|
|
2018
|
+
return True
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _wait_outline_transfer(handler, token, record):
|
|
2022
|
+
"""Wait only to the shared build deadline, waking for expiry/disconnect."""
|
|
2023
|
+
while True:
|
|
2024
|
+
if _request_peer_gone(handler):
|
|
2025
|
+
return False
|
|
2026
|
+
if record.ready.is_set():
|
|
2027
|
+
if record.failure_status is not None:
|
|
2028
|
+
handler._respond_json(
|
|
2029
|
+
record.failure_status, record.failure_body)
|
|
2030
|
+
return False
|
|
2031
|
+
if record.cancelled.is_set() or record.payload is None:
|
|
2032
|
+
handler._respond_json(
|
|
2033
|
+
410, {"error": "outline transfer expired"})
|
|
2034
|
+
return False
|
|
2035
|
+
return True
|
|
2036
|
+
now = time.monotonic()
|
|
2037
|
+
if now - record.created > _OUTLINE_TRANSFER_TTL:
|
|
2038
|
+
_cancel_outline_transfer(token, claim=record)
|
|
2039
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
2040
|
+
return False
|
|
2041
|
+
if now >= record.deadline:
|
|
2042
|
+
_finish_outline_transfer_failure(
|
|
2043
|
+
token, record, 504,
|
|
2044
|
+
{"error": "outline transfer timed out"})
|
|
2045
|
+
handler._respond_json(
|
|
2046
|
+
record.failure_status or 410,
|
|
2047
|
+
record.failure_body or {"error": "outline transfer expired"})
|
|
2048
|
+
return False
|
|
2049
|
+
record.ready.wait(min(0.05, record.deadline - now))
|
|
2050
|
+
|
|
2051
|
+
|
|
1640
2052
|
def _handle_get_conversation_outline_transfer_impl(handler, path: str) -> None:
|
|
1641
2053
|
"""Serve one bounded chunk from an opaque, immutable outline transfer."""
|
|
1642
2054
|
if not handler._require_transcripts_allowed():
|
|
@@ -1655,42 +2067,33 @@ def _handle_get_conversation_outline_transfer_impl(handler, path: str) -> None:
|
|
|
1655
2067
|
return
|
|
1656
2068
|
else:
|
|
1657
2069
|
offset = int(values[0])
|
|
1658
|
-
claim = _claim_outline_transfer(token)
|
|
2070
|
+
claim = _claim_outline_transfer(token, admit=True)
|
|
1659
2071
|
if claim is None:
|
|
1660
2072
|
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1661
2073
|
return
|
|
1662
2074
|
action, record, builder = claim
|
|
2075
|
+
if action == "busy":
|
|
2076
|
+
handler._respond_json(503, {"error": "outline transfer busy"})
|
|
2077
|
+
return
|
|
1663
2078
|
if action == "build":
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
if state == "too_large":
|
|
1683
|
-
handler._respond_json(
|
|
1684
|
-
413, {"error": "outline exceeds progressive transfer limit"})
|
|
1685
|
-
return
|
|
1686
|
-
if state == "expired":
|
|
1687
|
-
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1688
|
-
return
|
|
1689
|
-
elif action == "wait":
|
|
1690
|
-
record.ready.wait(_OUTLINE_TRANSFER_TTL)
|
|
1691
|
-
if record.cancelled.is_set() or record.payload is None:
|
|
1692
|
-
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1693
|
-
return
|
|
2079
|
+
try:
|
|
2080
|
+
threading.Thread(
|
|
2081
|
+
target=_produce_outline_transfer,
|
|
2082
|
+
args=(token, record, builder, handler),
|
|
2083
|
+
name="cctally-outline-build", daemon=True,
|
|
2084
|
+
).start()
|
|
2085
|
+
except RuntimeError:
|
|
2086
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
2087
|
+
global _OUTLINE_TRANSFERS_PRODUCERS
|
|
2088
|
+
_OUTLINE_TRANSFERS_PRODUCERS -= 1
|
|
2089
|
+
_finish_outline_transfer_failure(
|
|
2090
|
+
token, record, 503, {"error": "outline transfer busy"})
|
|
2091
|
+
if action != "ready":
|
|
2092
|
+
try:
|
|
2093
|
+
if not _wait_outline_transfer(handler, token, record):
|
|
2094
|
+
return
|
|
2095
|
+
finally:
|
|
2096
|
+
_release_outline_transfer_waiter(token, record)
|
|
1694
2097
|
try:
|
|
1695
2098
|
body = _read_outline_transfer(token, offset)
|
|
1696
2099
|
except ValueError:
|
|
@@ -1749,6 +2152,33 @@ def _handle_get_conversation_prompts_impl(handler, path: str) -> None:
|
|
|
1749
2152
|
|
|
1750
2153
|
_CONV_EXPORT_SCOPES = ("all", "prompts", "chat", "recipe")
|
|
1751
2154
|
|
|
2155
|
+
#: The #850 M5 fail-closed body, shared by `/export?anonymize=1` and
|
|
2156
|
+
#: `/anon-map`. Both routes answer HTTP 409 with it while any Codex thread
|
|
2157
|
+
#: `cwd` is undecodable: a path the store cannot supply as a scrub token can
|
|
2158
|
+
#: appear in any transcript, so neither route may hand back bytes or a token
|
|
2159
|
+
#: map that only looks scrubbed.
|
|
2160
|
+
_ANON_UNAVAILABLE_STATUS = "anonymization_unavailable"
|
|
2161
|
+
_ANON_UNAVAILABLE_REMEDY = "cctally cache-sync --source codex --rebuild"
|
|
2162
|
+
|
|
2163
|
+
|
|
2164
|
+
def _anon_unavailable_body(undecodable_cwd_rows: int) -> dict:
|
|
2165
|
+
return {
|
|
2166
|
+
"status": _ANON_UNAVAILABLE_STATUS,
|
|
2167
|
+
"undecodable_cwd_rows": int(undecodable_cwd_rows),
|
|
2168
|
+
"remedy": _ANON_UNAVAILABLE_REMEDY,
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
|
|
2172
|
+
def _anon_ambiguous_body(ambiguous_cwd_rows: int) -> dict:
|
|
2173
|
+
"""Typed refusal for a scoped path whose account ownership is unproven."""
|
|
2174
|
+
return {
|
|
2175
|
+
"status": _ANON_UNAVAILABLE_STATUS,
|
|
2176
|
+
"ambiguous_cwd_rows": int(ambiguous_cwd_rows),
|
|
2177
|
+
"reason": "ambiguous_account_provenance",
|
|
2178
|
+
"remedy": "use a raw export or omit the account scope",
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
|
|
1752
2182
|
def _handle_get_conversation_export_impl(handler, path: str) -> None:
|
|
1753
2183
|
"""``GET /api/conversation/<sid>/export?scope=<all|prompts|chat|recipe>``
|
|
1754
2184
|
— whole-session Markdown (issue #217 S5 F1/F5).
|
|
@@ -1806,15 +2236,23 @@ def _handle_get_conversation_export_impl(handler, path: str) -> None:
|
|
|
1806
2236
|
cq = handler._conversation_query()
|
|
1807
2237
|
anon = sys.modules["cctally"]._load_sibling("_lib_conversation_anon")
|
|
1808
2238
|
srcs = {cref.source} if cref else set()
|
|
1809
|
-
|
|
2239
|
+
result = cq.build_anon_plan_for_sources(
|
|
1810
2240
|
conn, home_dir=_os.path.expanduser("~"), sources=srcs)
|
|
1811
|
-
|
|
2241
|
+
if result.undecodable_cwd_rows:
|
|
2242
|
+
return _anon_unavailable_body(result.undecodable_cwd_rows)
|
|
2243
|
+
if result.ambiguous_cwd_rows:
|
|
2244
|
+
return _anon_ambiguous_body(result.ambiguous_cwd_rows)
|
|
2245
|
+
return {**env,
|
|
2246
|
+
"markdown": anon.scrub_text(env["markdown"], result.plan)}
|
|
1812
2247
|
|
|
1813
2248
|
ok, env = handler._run_conversation_query(
|
|
1814
2249
|
_q_kernel, "/api/conversation/export")
|
|
1815
2250
|
if not ok:
|
|
1816
2251
|
return
|
|
1817
2252
|
status = (env or {}).get("status")
|
|
2253
|
+
if status == _ANON_UNAVAILABLE_STATUS:
|
|
2254
|
+
handler._respond_json(409, env)
|
|
2255
|
+
return
|
|
1818
2256
|
if status == "ok":
|
|
1819
2257
|
data = env["markdown"].encode("utf-8")
|
|
1820
2258
|
handler.send_response(200)
|
|
@@ -1896,14 +2334,22 @@ def _handle_get_conversation_anon_map_impl(handler, path: str) -> None:
|
|
|
1896
2334
|
exists = cq.conversation_exists(conn, cref.native_key)
|
|
1897
2335
|
if not exists:
|
|
1898
2336
|
return None
|
|
1899
|
-
|
|
2337
|
+
result = cq.build_anon_plan_for_sources(
|
|
1900
2338
|
conn, home_dir=_os.path.expanduser("~"), sources={cref.source})
|
|
1901
|
-
|
|
2339
|
+
if result.undecodable_cwd_rows:
|
|
2340
|
+
return _anon_unavailable_body(result.undecodable_cwd_rows)
|
|
2341
|
+
if result.ambiguous_cwd_rows:
|
|
2342
|
+
return _anon_ambiguous_body(result.ambiguous_cwd_rows)
|
|
2343
|
+
return anon.plan_to_wire(result.plan)
|
|
1902
2344
|
|
|
1903
2345
|
ok, body = handler._run_conversation_query(
|
|
1904
2346
|
_q_kernel, "/api/conversation/anon-map")
|
|
1905
2347
|
if not ok:
|
|
1906
2348
|
return
|
|
2349
|
+
if isinstance(body, dict) and body.get(
|
|
2350
|
+
"status") == _ANON_UNAVAILABLE_STATUS:
|
|
2351
|
+
handler._respond_json(409, body)
|
|
2352
|
+
return
|
|
1907
2353
|
if body is None:
|
|
1908
2354
|
handler._respond_json(
|
|
1909
2355
|
404, {"status": "not_found", "conversation_key": session_id})
|