cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
|
@@ -15,6 +15,7 @@ import base64 as _base64
|
|
|
15
15
|
import json as _json
|
|
16
16
|
import os
|
|
17
17
|
import re
|
|
18
|
+
import dataclasses
|
|
18
19
|
import sqlite3
|
|
19
20
|
from datetime import datetime as _datetime, timezone as _timezone
|
|
20
21
|
|
|
@@ -1203,22 +1204,47 @@ def _rollup_authoritative(conn) -> bool:
|
|
|
1203
1204
|
durable ``conversation_sessions_backfill_pending`` flag is NOT set, so the
|
|
1204
1205
|
rollup has been fully recomputed and the fast read is safe.
|
|
1205
1206
|
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1207
|
+
FAILS CLOSED on a read it could not perform (#769 S3, the read side of
|
|
1208
|
+
#728). It used to `except sqlite3.OperationalError: return True`, so a
|
|
1209
|
+
locked or otherwise unreadable store reported the rollup as authoritative
|
|
1210
|
+
and the rail served a rollup nothing had verified. That is the same failure
|
|
1211
|
+
class the writer sites had, in the more dangerous direction: a refused
|
|
1212
|
+
write costs a stale rollup, while a wrongly-authoritative read costs a
|
|
1213
|
+
wrong answer. A read that did not happen now reports "not authoritative",
|
|
1214
|
+
which costs live aggregation.
|
|
1215
|
+
|
|
1216
|
+
The one carve-out is structural and matches
|
|
1217
|
+
`_cctally_cache._read_pricing_fingerprint_observation` exactly: a store
|
|
1218
|
+
whose ``sqlite_master`` probe positively shows no ``cache_meta`` table
|
|
1219
|
+
determinately holds no pending flag, so it stays authoritative. That keeps
|
|
1220
|
+
every bare / in-memory / schema-not-applied connection on the fast path,
|
|
1221
|
+
which is the case the old blanket True was actually protecting. The probe
|
|
1222
|
+
itself fails closed: a lock that defeats the flag SELECT defeats the probe
|
|
1223
|
+
too, so an unanswerable probe is a failed read, not an absent table."""
|
|
1212
1224
|
try:
|
|
1213
1225
|
pending = conn.execute(
|
|
1214
1226
|
"SELECT 1 FROM cache_meta "
|
|
1215
1227
|
"WHERE key='conversation_sessions_backfill_pending'"
|
|
1216
1228
|
).fetchone() is not None
|
|
1217
1229
|
except sqlite3.OperationalError:
|
|
1218
|
-
return
|
|
1230
|
+
return not _cache_meta_table_present(conn)
|
|
1219
1231
|
return not pending
|
|
1220
1232
|
|
|
1221
1233
|
|
|
1234
|
+
def _cache_meta_table_present(conn) -> bool:
|
|
1235
|
+
"""Whether ``cache_meta`` exists, answering False only on a probe that
|
|
1236
|
+
succeeded and found nothing. An unanswerable probe reports True, so its
|
|
1237
|
+
caller treats the store as one it failed to read rather than one that has
|
|
1238
|
+
nowhere to record state."""
|
|
1239
|
+
try:
|
|
1240
|
+
return conn.execute(
|
|
1241
|
+
"SELECT 1 FROM sqlite_master "
|
|
1242
|
+
"WHERE type='table' AND name='cache_meta'"
|
|
1243
|
+
).fetchone() is not None
|
|
1244
|
+
except sqlite3.OperationalError:
|
|
1245
|
+
return True
|
|
1246
|
+
|
|
1247
|
+
|
|
1222
1248
|
# --- Browse-list filter predicates (spec §2) ------------------------------
|
|
1223
1249
|
# The rail filters on four axes (date / project / cost / cache-rebuilds). All
|
|
1224
1250
|
# four are STORED columns on the conversation_sessions rollup, so the rollup
|
|
@@ -1492,12 +1518,26 @@ def list_conversation_facets(conn) -> dict:
|
|
|
1492
1518
|
|
|
1493
1519
|
Returns ``{"projects": [{"project_label", "count"}, ...],
|
|
1494
1520
|
"models": [{"family", "count"}, ...]}``."""
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1521
|
+
# #717. `projects` is read from the rollup, so while the rollup is being
|
|
1522
|
+
# rebuilt this list is empty for a reason no client can infer from an
|
|
1523
|
+
# empty array — and "no projects" and "the projects are not indexed yet"
|
|
1524
|
+
# are different facts that call for different words in the popover.
|
|
1525
|
+
#
|
|
1526
|
+
# The authority is `_rollup_authoritative` — the absence of
|
|
1527
|
+
# `conversation_sessions_backfill_pending` — and explicitly NOT the
|
|
1528
|
+
# pricing refusal latch: a rebuild pre-clear refusal coexists with an
|
|
1529
|
+
# intact authoritative rollup and arms no backfill flag, so keying on it
|
|
1530
|
+
# would degrade a rail that is perfectly correct.
|
|
1531
|
+
degraded = not _rollup_authoritative(conn)
|
|
1532
|
+
if degraded:
|
|
1533
|
+
projects: list = []
|
|
1534
|
+
else:
|
|
1535
|
+
rows = conn.execute(
|
|
1536
|
+
"SELECT project_label, COUNT(*) FROM conversation_sessions "
|
|
1537
|
+
"WHERE project_label IS NOT NULL AND project_label != '' "
|
|
1538
|
+
"GROUP BY project_label ORDER BY project_label"
|
|
1539
|
+
).fetchall()
|
|
1540
|
+
projects = [{"project_label": p, "count": n} for p, n in rows]
|
|
1501
1541
|
|
|
1502
1542
|
fam_sessions: dict = {}
|
|
1503
1543
|
for sid, model in conn.execute(
|
|
@@ -1513,7 +1553,15 @@ def list_conversation_facets(conn) -> dict:
|
|
|
1513
1553
|
for fam in _MODEL_FAMILY_ORDER
|
|
1514
1554
|
if fam in fam_sessions
|
|
1515
1555
|
]
|
|
1516
|
-
|
|
1556
|
+
# `models` keeps its live counts in both states: it folds
|
|
1557
|
+
# `conversation_messages` directly and never touches the rollup, so a
|
|
1558
|
+
# pending backfill does not make it wrong. The flag is OMITTED on the
|
|
1559
|
+
# authoritative path, so the ordinary envelope is byte-identical to what
|
|
1560
|
+
# every existing consumer already parses.
|
|
1561
|
+
facets = {"projects": projects, "models": models}
|
|
1562
|
+
if degraded:
|
|
1563
|
+
facets["filter_degraded"] = True
|
|
1564
|
+
return facets
|
|
1517
1565
|
|
|
1518
1566
|
|
|
1519
1567
|
def _selected_conversation_summary(conn, session_id):
|
|
@@ -3104,6 +3152,188 @@ def build_anon_plan_for_db(conn, *, home_dir):
|
|
|
3104
3152
|
home_dirs=sorted(home_dirs), usernames=sorted(usernames))
|
|
3105
3153
|
|
|
3106
3154
|
|
|
3155
|
+
#: The Codex leg's fail-closed count, read from the RAW threads table so a
|
|
3156
|
+
#: scoped plan observes the same condition an unscoped one does. The vocabulary
|
|
3157
|
+
#: statements below keep their unqualified names for the legacy/unscoped path;
|
|
3158
|
+
#: account-scoped requests use physical event provenance before reading the
|
|
3159
|
+
#: same raw table.
|
|
3160
|
+
_CODEX_UNDECODABLE_CWD_SQL = """
|
|
3161
|
+
SELECT CAST(cwd AS BLOB) AS cwd_blob
|
|
3162
|
+
FROM {threads}
|
|
3163
|
+
"""
|
|
3164
|
+
|
|
3165
|
+
|
|
3166
|
+
@dataclasses.dataclass(frozen=True)
|
|
3167
|
+
class SourceAnonPlan:
|
|
3168
|
+
"""A provider-aware :class:`AnonPlan` and the fail-closed count beside it.
|
|
3169
|
+
|
|
3170
|
+
``undecodable_cwd_rows`` counts Codex threads whose ``cwd`` bytes are not
|
|
3171
|
+
valid UTF-8. A path the store cannot supply as a scrub token can appear in
|
|
3172
|
+
any transcript, so a count above zero refuses EVERY anonymized export whose
|
|
3173
|
+
plan includes the Codex leg, rather than only the conversation that owns
|
|
3174
|
+
the bad row (#850 M5). ``--raw``, ``search`` and Claude-only plans are
|
|
3175
|
+
unaffected. ``ambiguous_cwd_rows`` counts scoped Codex thread/root paths
|
|
3176
|
+
whose account ownership is not proven by physical account-stamped events;
|
|
3177
|
+
scoped anonymized copies refuse when it is non-zero.
|
|
3178
|
+
"""
|
|
3179
|
+
|
|
3180
|
+
plan: object
|
|
3181
|
+
undecodable_cwd_rows: int
|
|
3182
|
+
ambiguous_cwd_rows: int = 0
|
|
3183
|
+
|
|
3184
|
+
|
|
3185
|
+
def _anon_scope_account(conn) -> str | None:
|
|
3186
|
+
"""Return the active read scope without confusing it with an empty store."""
|
|
3187
|
+
scoped = conn.execute(
|
|
3188
|
+
"SELECT 1 FROM sqlite_temp_master "
|
|
3189
|
+
"WHERE type='table' AND name='_conversation_account_scope'"
|
|
3190
|
+
).fetchone()
|
|
3191
|
+
if scoped is None:
|
|
3192
|
+
return None
|
|
3193
|
+
row = conn.execute(
|
|
3194
|
+
"SELECT account_key FROM temp._conversation_account_scope"
|
|
3195
|
+
).fetchone()
|
|
3196
|
+
if row is None:
|
|
3197
|
+
raise RuntimeError("conversation account scope has no account key")
|
|
3198
|
+
return row[0]
|
|
3199
|
+
|
|
3200
|
+
|
|
3201
|
+
def _scoped_codex_anon_paths(conn, account_key: str):
|
|
3202
|
+
"""Prove scoped paths from account-stamped physical session_meta events.
|
|
3203
|
+
|
|
3204
|
+
The thread row and provider root are conversation/root metadata, not account
|
|
3205
|
+
facts. In particular, a shared thread's CWD can have been written before an
|
|
3206
|
+
account switch. A session_meta event's own account stamp is the narrowest
|
|
3207
|
+
retained proof for its payload CWD and source root. A path/root seen under
|
|
3208
|
+
another session_meta account is ambiguous and is never offered as a
|
|
3209
|
+
client-side token.
|
|
3210
|
+
"""
|
|
3211
|
+
from _lib_codex_metadata import (
|
|
3212
|
+
decode_codex_project_metadata, resolve_codex_threads_table,
|
|
3213
|
+
)
|
|
3214
|
+
|
|
3215
|
+
observations: dict[str, set[str]] = {}
|
|
3216
|
+
root_owners: dict[str, set[str]] = {}
|
|
3217
|
+
# Only session_meta carries the CWD/root provenance needed here. Keep this
|
|
3218
|
+
# read narrow: selecting and decoding every response/tool event made each
|
|
3219
|
+
# account-scoped copy proportional to the entire physical event store.
|
|
3220
|
+
# A read-only opener that predates the schema ensure must refuse the
|
|
3221
|
+
# vocabulary rather than silently reintroduce the old full-table scan.
|
|
3222
|
+
# The raw thread pass below will turn every visible path into an ambiguity
|
|
3223
|
+
# refusal while still preserving the independent undecodable count.
|
|
3224
|
+
provenance_index = conn.execute(
|
|
3225
|
+
"SELECT 1 FROM main.sqlite_master "
|
|
3226
|
+
"WHERE type='index' AND name='idx_codex_events_session_meta_provenance'"
|
|
3227
|
+
).fetchone() is not None
|
|
3228
|
+
if provenance_index:
|
|
3229
|
+
for source_root_key, payload_json, owner in conn.execute(
|
|
3230
|
+
"SELECT source_root_key, payload_json, "
|
|
3231
|
+
"COALESCE(account_key,'unattributed') "
|
|
3232
|
+
"FROM main.codex_conversation_events "
|
|
3233
|
+
"WHERE record_type='session_meta'"
|
|
3234
|
+
):
|
|
3235
|
+
if source_root_key:
|
|
3236
|
+
root_owners.setdefault(source_root_key, set()).add(owner)
|
|
3237
|
+
try:
|
|
3238
|
+
obj = _json.loads(payload_json)
|
|
3239
|
+
except (TypeError, ValueError):
|
|
3240
|
+
continue
|
|
3241
|
+
payload = obj.get("payload") if isinstance(obj, dict) else None
|
|
3242
|
+
path = payload.get("cwd") if isinstance(payload, dict) else None
|
|
3243
|
+
if isinstance(path, str) and path and os.path.isabs(path):
|
|
3244
|
+
observations.setdefault(path, set()).add(owner)
|
|
3245
|
+
visible_keys = {
|
|
3246
|
+
key for (key,) in conn.execute(
|
|
3247
|
+
"SELECT DISTINCT conversation_key FROM codex_conversation_messages"
|
|
3248
|
+
) if key
|
|
3249
|
+
}
|
|
3250
|
+
try:
|
|
3251
|
+
threads = resolve_codex_threads_table(conn)
|
|
3252
|
+
except RuntimeError:
|
|
3253
|
+
if visible_keys:
|
|
3254
|
+
# Messages without their raw thread metadata leave project-path
|
|
3255
|
+
# coverage unknown. Refuse rather than offer a partial scrub plan.
|
|
3256
|
+
raise RuntimeError("Codex raw threads table missing for scoped anonymization")
|
|
3257
|
+
return set(), 0, 0
|
|
3258
|
+
ambiguous = 0
|
|
3259
|
+
thread_schema = threads.split(".", 1)[0]
|
|
3260
|
+
root_paths: dict[str, str] = {}
|
|
3261
|
+
for root_key, root_path in conn.execute(
|
|
3262
|
+
"SELECT source_root_key, canonical_root_path FROM "
|
|
3263
|
+
+ thread_schema + ".codex_source_roots"
|
|
3264
|
+
):
|
|
3265
|
+
if isinstance(root_key, str) and isinstance(root_path, str):
|
|
3266
|
+
if root_path and os.path.isabs(root_path):
|
|
3267
|
+
root_paths[root_key] = root_path
|
|
3268
|
+
owned_roots = {
|
|
3269
|
+
root_key for root_key, owners in root_owners.items()
|
|
3270
|
+
if owners == {account_key}
|
|
3271
|
+
}
|
|
3272
|
+
# A string can be both a provider root and a different account's project
|
|
3273
|
+
# CWD. Merge evidence by path before admitting it; unioning individually
|
|
3274
|
+
# approved roots back into `owned` would leak the foreign CWD to anon-map.
|
|
3275
|
+
path_owners = {path: set(owners) for path, owners in observations.items()}
|
|
3276
|
+
for root_key, root_path in root_paths.items():
|
|
3277
|
+
path_owners.setdefault(root_path, set()).update(
|
|
3278
|
+
root_owners.get(root_key, ())
|
|
3279
|
+
)
|
|
3280
|
+
owned = {path for path, owners in path_owners.items()
|
|
3281
|
+
if owners == {account_key}}
|
|
3282
|
+
|
|
3283
|
+
undecodable = 0
|
|
3284
|
+
for key, source_root_key, cwd_blob in conn.execute(
|
|
3285
|
+
"SELECT conversation_key, source_root_key, CAST(cwd AS BLOB) "
|
|
3286
|
+
"FROM " + threads
|
|
3287
|
+
):
|
|
3288
|
+
decoded = decode_codex_project_metadata(cwd_blob, None)
|
|
3289
|
+
if "cwd" in decoded.undecodable_fields:
|
|
3290
|
+
undecodable += 1
|
|
3291
|
+
if key not in visible_keys:
|
|
3292
|
+
continue
|
|
3293
|
+
cwd = decoded.cwd
|
|
3294
|
+
if isinstance(cwd, str) and cwd and os.path.isabs(cwd):
|
|
3295
|
+
if cwd not in owned:
|
|
3296
|
+
ambiguous += 1
|
|
3297
|
+
if source_root_key and source_root_key in root_paths:
|
|
3298
|
+
if (source_root_key not in owned_roots
|
|
3299
|
+
or root_paths[source_root_key] not in owned):
|
|
3300
|
+
ambiguous += 1
|
|
3301
|
+
return owned, ambiguous, undecodable
|
|
3302
|
+
|
|
3303
|
+
|
|
3304
|
+
def _count_undecodable_codex_cwd(conn) -> int:
|
|
3305
|
+
"""Codex threads whose ``cwd`` cannot be decoded, read past a scoped view.
|
|
3306
|
+
|
|
3307
|
+
A store with no Codex threads table has no undecodable row, so it answers
|
|
3308
|
+
zero rather than refusing an export that would otherwise have worked. That
|
|
3309
|
+
absent-table case reaches here as ``resolve_codex_threads_table``'s
|
|
3310
|
+
``RuntimeError``, and it is the ONLY failure this answers with zero.
|
|
3311
|
+
|
|
3312
|
+
A ``sqlite3.Error`` is a read the store could not perform, not the answer
|
|
3313
|
+
"nothing is undecodable", so it propagates. M5 is fail-closed: a locked or
|
|
3314
|
+
malformed cache.db must stop every anonymized export rather than let one
|
|
3315
|
+
proceed on a scrub vocabulary that was never established. The escaping
|
|
3316
|
+
error already lands on each caller's own error path — ``cctally transcript
|
|
3317
|
+
export`` reports it and exits non-zero without writing the transcript, and
|
|
3318
|
+
``_run_conversation_query`` answers both dashboard routes with a typed JSON
|
|
3319
|
+
error on a 500, emitting neither markdown nor a token map.
|
|
3320
|
+
"""
|
|
3321
|
+
from _lib_codex_metadata import (
|
|
3322
|
+
decode_codex_project_metadata, resolve_codex_threads_table,
|
|
3323
|
+
)
|
|
3324
|
+
|
|
3325
|
+
try:
|
|
3326
|
+
threads = resolve_codex_threads_table(conn)
|
|
3327
|
+
except RuntimeError:
|
|
3328
|
+
return 0
|
|
3329
|
+
rows = conn.execute(_CODEX_UNDECODABLE_CWD_SQL.format(threads=threads))
|
|
3330
|
+
count = 0
|
|
3331
|
+
for (cwd_blob,) in rows:
|
|
3332
|
+
if "cwd" in decode_codex_project_metadata(cwd_blob, None).undecodable_fields:
|
|
3333
|
+
count += 1
|
|
3334
|
+
return count
|
|
3335
|
+
|
|
3336
|
+
|
|
3107
3337
|
def build_anon_plan_for_sources(conn, *, home_dir, sources):
|
|
3108
3338
|
"""Provider-aware :class:`AnonPlan` for QUALIFIED conversation surfaces
|
|
3109
3339
|
(#294 S7, §3.6). A SEPARATE builder from :func:`build_anon_plan_for_db`, which
|
|
@@ -3113,16 +3343,30 @@ def build_anon_plan_for_sources(conn, *, home_dir, sources):
|
|
|
3113
3343
|
requests (both providers), which are new and carry no byte-compat debt.
|
|
3114
3344
|
|
|
3115
3345
|
``sources`` selects the provider legs to include (e.g. ``{"codex"}`` or
|
|
3116
|
-
``{"claude"}
|
|
3117
|
-
provider root dirs from ``codex_source_roots.canonical_root_path``,
|
|
3118
|
-
project paths from validated ``codex_conversation_threads.cwd``
|
|
3119
|
-
only), display labels from
|
|
3120
|
-
``
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3346
|
+
``{"claude"}`). An unscoped Codex leg draws ONLY from the authoritative
|
|
3347
|
+
tables: provider root dirs from ``codex_source_roots.canonical_root_path``,
|
|
3348
|
+
observed project paths from validated ``codex_conversation_threads.cwd``
|
|
3349
|
+
(absolute paths only), and display labels from
|
|
3350
|
+
``codex_conversation_rollups.project_label``. An account-scoped Codex leg
|
|
3351
|
+
instead accepts a path only after account-stamped physical events prove its
|
|
3352
|
+
ownership; the conversation-level thread/root metadata is then used only to
|
|
3353
|
+
detect an unproven path that requires refusal. ``git_json`` is NOT a path
|
|
3354
|
+
source (its fields are branch/repository metadata, not filesystem roots).
|
|
3355
|
+
The existing home-dir/username machinery and ``SECRET_PATTERNS`` apply
|
|
3356
|
+
unchanged; coverage stays best-effort-over-known-tokens and the per-token
|
|
3357
|
+
replacement stays fail-closed.
|
|
3358
|
+
|
|
3359
|
+
Returns a :class:`SourceAnonPlan`: the plan, the number of Codex thread
|
|
3360
|
+
``cwd`` values the store cannot decode, and (for an account-scoped request)
|
|
3361
|
+
the number of visible thread/root paths without physical ownership proof.
|
|
3362
|
+
The Codex leg decodes PER ROW, so a single undecodable value no longer costs
|
|
3363
|
+
the vocabulary every path after it — the loop's
|
|
3364
|
+
``except sqlite3.OperationalError: pass`` swallowed that error
|
|
3365
|
+
mid-iteration and the plan came back silently short."""
|
|
3366
|
+
from _lib_codex_metadata import decode_codex_project_metadata
|
|
3124
3367
|
from _lib_conversation_anon import build_anon_plan
|
|
3125
3368
|
want = set(sources or ())
|
|
3369
|
+
scoped_account = _anon_scope_account(conn)
|
|
3126
3370
|
project_roots: dict = {}
|
|
3127
3371
|
paths: set = set()
|
|
3128
3372
|
if "claude" in want:
|
|
@@ -3146,33 +3390,49 @@ def build_anon_plan_for_sources(conn, *, home_dir, sources):
|
|
|
3146
3390
|
for c in cwds:
|
|
3147
3391
|
project_roots[c] = _project_label(c)
|
|
3148
3392
|
paths.add(c)
|
|
3393
|
+
undecodable_cwd_rows = 0
|
|
3394
|
+
ambiguous_cwd_rows = 0
|
|
3149
3395
|
if "codex" in want:
|
|
3150
|
-
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3396
|
+
if scoped_account is not None:
|
|
3397
|
+
(
|
|
3398
|
+
owned,
|
|
3399
|
+
ambiguous_cwd_rows,
|
|
3400
|
+
undecodable_cwd_rows,
|
|
3401
|
+
) = _scoped_codex_anon_paths(conn, scoped_account)
|
|
3402
|
+
for cwd in owned:
|
|
3403
|
+
project_roots[cwd] = _project_label(cwd)
|
|
3404
|
+
paths.add(cwd)
|
|
3405
|
+
else:
|
|
3406
|
+
labels: dict = {}
|
|
3407
|
+
try:
|
|
3408
|
+
for ck, pl in conn.execute(
|
|
3409
|
+
"SELECT conversation_key, project_label FROM codex_conversation_rollups "
|
|
3410
|
+
"WHERE project_label IS NOT NULL AND project_label != ''"):
|
|
3411
|
+
labels[ck] = pl
|
|
3412
|
+
except sqlite3.OperationalError:
|
|
3413
|
+
pass
|
|
3414
|
+
try:
|
|
3415
|
+
for ck, cwd_blob in conn.execute(
|
|
3416
|
+
"SELECT conversation_key, CAST(cwd AS BLOB) AS cwd_blob "
|
|
3417
|
+
"FROM codex_conversation_threads "
|
|
3418
|
+
"WHERE cwd IS NOT NULL AND cwd != ''"):
|
|
3419
|
+
cwd = decode_codex_project_metadata(cwd_blob, None).cwd
|
|
3420
|
+
if isinstance(cwd, str) and cwd and os.path.isabs(cwd):
|
|
3421
|
+
project_roots[cwd] = labels.get(ck) or _project_label(cwd)
|
|
3422
|
+
paths.add(cwd)
|
|
3423
|
+
except sqlite3.OperationalError:
|
|
3424
|
+
pass
|
|
3425
|
+
if scoped_account is None:
|
|
3426
|
+
undecodable_cwd_rows = _count_undecodable_codex_cwd(conn)
|
|
3427
|
+
try:
|
|
3428
|
+
for (root_path,) in conn.execute(
|
|
3429
|
+
"SELECT canonical_root_path FROM codex_source_roots "
|
|
3430
|
+
"WHERE canonical_root_path IS NOT NULL AND canonical_root_path != ''"):
|
|
3431
|
+
if isinstance(root_path, str) and root_path and os.path.isabs(root_path):
|
|
3432
|
+
project_roots.setdefault(root_path, _project_label(root_path))
|
|
3433
|
+
paths.add(root_path)
|
|
3434
|
+
except sqlite3.OperationalError:
|
|
3435
|
+
pass
|
|
3176
3436
|
home_dirs: set = set()
|
|
3177
3437
|
usernames: set = set()
|
|
3178
3438
|
if home_dir:
|
|
@@ -3187,9 +3447,13 @@ def build_anon_plan_for_sources(conn, *, home_dir, sources):
|
|
|
3187
3447
|
if user:
|
|
3188
3448
|
home_dirs.add(prefix + user)
|
|
3189
3449
|
usernames.add(user)
|
|
3190
|
-
return
|
|
3191
|
-
|
|
3192
|
-
|
|
3450
|
+
return SourceAnonPlan(
|
|
3451
|
+
plan=build_anon_plan(
|
|
3452
|
+
project_roots=project_roots,
|
|
3453
|
+
home_dirs=sorted(home_dirs), usernames=sorted(usernames)),
|
|
3454
|
+
undecodable_cwd_rows=undecodable_cwd_rows,
|
|
3455
|
+
ambiguous_cwd_rows=ambiguous_cwd_rows,
|
|
3456
|
+
)
|
|
3193
3457
|
|
|
3194
3458
|
|
|
3195
3459
|
_TASK_TRIO = ("TaskCreate", "TaskUpdate", "TaskList")
|