cctally 1.108.0 → 1.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +6 -6
- package/bin/_cctally_account.py +4 -0
- package/bin/_cctally_alerts.py +6 -2
- package/bin/_cctally_cache.py +1652 -166
- package/bin/_cctally_config.py +3 -2
- package/bin/_cctally_core.py +594 -33
- package/bin/_cctally_dashboard.py +1585 -201
- package/bin/_cctally_dashboard_conversation.py +494 -48
- package/bin/_cctally_dashboard_envelope.py +167 -11
- package/bin/_cctally_dashboard_share.py +154 -36
- package/bin/_cctally_dashboard_sources.py +2615 -422
- package/bin/_cctally_db.py +678 -5
- package/bin/_cctally_diagnosis_sources.py +238 -26
- package/bin/_cctally_doctor.py +99 -18
- package/bin/_cctally_five_hour.py +1295 -40
- package/bin/_cctally_forecast.py +124 -28
- package/bin/_cctally_journal.py +408 -38
- package/bin/_cctally_milestone_history.py +150 -14
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_percent_breakdown.py +6 -1
- package/bin/_cctally_project.py +48 -76
- package/bin/_cctally_quota.py +39 -7
- package/bin/_cctally_quota_model.py +17 -2
- package/bin/_cctally_record.py +1647 -443
- package/bin/_cctally_rederive.py +899 -30
- package/bin/_cctally_refresh.py +2 -1
- package/bin/_cctally_share.py +24 -5
- package/bin/_cctally_source_analytics.py +1035 -78
- package/bin/_cctally_statusline.py +83 -19
- package/bin/_cctally_store.py +61 -4
- package/bin/_cctally_transcript.py +27 -2
- package/bin/_cctally_tui.py +264 -76
- package/bin/_cctally_weekrefs.py +28 -5
- package/bin/_lib_blocks.py +127 -20
- package/bin/_lib_cache_report.py +20 -5
- package/bin/_lib_codex_conversation_query.py +297 -64
- package/bin/_lib_codex_hooks.py +102 -0
- package/bin/_lib_codex_metadata.py +207 -0
- package/bin/_lib_conversation.py +119 -0
- package/bin/_lib_conversation_dispatch.py +31 -1
- package/bin/_lib_conversation_query.py +315 -51
- package/bin/_lib_conversation_retention.py +413 -25
- package/bin/_lib_dashboard_sources.py +267 -7
- package/bin/_lib_diagnosis.py +10 -0
- package/bin/_lib_diff_kernel.py +12 -1
- package/bin/_lib_doctor.py +230 -8
- package/bin/_lib_ingest_frontier.py +1366 -49
- package/bin/_lib_journal.py +19 -2
- package/bin/_lib_merge_gate.py +307 -0
- package/bin/_lib_pricing.py +588 -29
- package/bin/_lib_record.py +130 -12
- package/bin/_lib_rederive.py +903 -2
- package/bin/_lib_render.py +9 -3
- package/bin/_lib_retained_size.py +27 -2
- package/bin/_lib_segment_summary.py +66 -8
- package/bin/_lib_share_templates.py +28 -18
- package/bin/_lib_snapshot_cache.py +115 -1
- package/bin/_lib_source_retry.py +320 -0
- package/bin/_lib_statusline_candidates.py +400 -36
- package/bin/_lib_tick_stats.py +119 -0
- package/bin/_lib_view_models.py +256 -76
- package/bin/cctally +69 -0
- package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
- package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
- package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
- package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
- package/dashboard/static/assets/index-COhxe3vU.css +1 -0
- package/dashboard/static/assets/index-DdU62sov.js +13 -0
- package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
- package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
- package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
- package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
- package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
- package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
- package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
- package/dashboard/static/assets/index-CIWsbux3.js +0 -13
- package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
- package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
package/bin/_lib_codex_hooks.py
CHANGED
|
@@ -535,6 +535,84 @@ def observe_codex_hook_root(root: "CodexHookRoot") -> CodexHookObservation:
|
|
|
535
535
|
)
|
|
536
536
|
|
|
537
537
|
|
|
538
|
+
def codex_configuration_generation(roots) -> "str | None":
|
|
539
|
+
"""One reproducible digest of every configured root's hook configuration.
|
|
540
|
+
|
|
541
|
+
#769 S6, #716 Task A. Returns a lowercase SHA-256 hex digest, or ``None``
|
|
542
|
+
when the configuration cannot be read or is ambiguous.
|
|
543
|
+
|
|
544
|
+
WHAT IT COVERS, per root, in canonical root order: the root's identity, the
|
|
545
|
+
exact bytes of ``hooks.json``, the exact bytes of ``config.toml``, and the
|
|
546
|
+
managed event and command identity — the event name, the slot position and
|
|
547
|
+
the command vector of the one managed handler on each event.
|
|
548
|
+
|
|
549
|
+
WHY A DIGEST RATHER THAN A RECORD. The value is stamped into a ticket that
|
|
550
|
+
the dashboard's diagnostic surfaces can reach, and a stored record would
|
|
551
|
+
put filesystem paths and configuration contents there. A digest answers
|
|
552
|
+
the only question the frontier asks — is the configuration the same one? —
|
|
553
|
+
and answers nothing else.
|
|
554
|
+
|
|
555
|
+
WHY BYTES RATHER THAN MTIMES. The classification in
|
|
556
|
+
``_observe_classified_state`` compares ``hooks.json`` and ``config.toml``
|
|
557
|
+
mtimes precisely because Codex's recorded ``trusted_hash`` cannot be
|
|
558
|
+
reproduced (§1.5). A bare ``touch`` therefore retires trust that nothing
|
|
559
|
+
changed, and a rewrite that preserves the mtime does not retire trust that
|
|
560
|
+
did. Hashing the bytes has neither failure.
|
|
561
|
+
|
|
562
|
+
UNREADABLE AND AMBIGUOUS BOTH RETURN ``None``, and ``None`` never matches
|
|
563
|
+
any stamped generation, so both stay untrusted. An empty root set is one
|
|
564
|
+
of them: nothing was configured, so nothing can be shown to have run.
|
|
565
|
+
"""
|
|
566
|
+
import hashlib # noqa: PLC0415 — stdlib, matching the kernels beside it
|
|
567
|
+
|
|
568
|
+
roots = list(roots)
|
|
569
|
+
if not roots:
|
|
570
|
+
return None
|
|
571
|
+
digest = hashlib.sha256()
|
|
572
|
+
for root in sorted(roots, key=lambda item: str(item.hooks_path)):
|
|
573
|
+
try:
|
|
574
|
+
document = _read_hooks_document(root.hooks_path)
|
|
575
|
+
hooks_bytes = root.hooks_path.read_bytes()
|
|
576
|
+
except (CodexHooksError, OSError):
|
|
577
|
+
return None
|
|
578
|
+
try:
|
|
579
|
+
config_bytes = codex_config_path(root).read_bytes()
|
|
580
|
+
except FileNotFoundError:
|
|
581
|
+
# An absent config.toml is a real, ordinary state: Codex writes it
|
|
582
|
+
# when it first records trust. Distinguish it from empty rather
|
|
583
|
+
# than refusing, so a root that has never been trusted still has a
|
|
584
|
+
# stable generation.
|
|
585
|
+
config_bytes = b""
|
|
586
|
+
except OSError:
|
|
587
|
+
return None
|
|
588
|
+
digest.update(b"root\x00")
|
|
589
|
+
digest.update(str(root.source_root_key).encode("utf-8"))
|
|
590
|
+
digest.update(b"\x00hooks\x00")
|
|
591
|
+
digest.update(hooks_bytes)
|
|
592
|
+
digest.update(b"\x00config\x00")
|
|
593
|
+
digest.update(config_bytes)
|
|
594
|
+
for event in CODEX_HOOK_EVENTS:
|
|
595
|
+
managed = [
|
|
596
|
+
(group_index, handler_index, handler)
|
|
597
|
+
for group_index, handler_index, handler
|
|
598
|
+
in iter_codex_hook_slots(document, event)
|
|
599
|
+
if is_managed_codex_hook_handler(handler)
|
|
600
|
+
]
|
|
601
|
+
if len(managed) != 1:
|
|
602
|
+
# Zero is "not installed" and more than one is the duplicate
|
|
603
|
+
# `setup` exists to reconcile. Neither is a configuration a
|
|
604
|
+
# ticket could have been written under unambiguously.
|
|
605
|
+
return None
|
|
606
|
+
group_index, handler_index, handler = managed[0]
|
|
607
|
+
tokens = _handler_command_tokens(handler) or []
|
|
608
|
+
digest.update(b"\x00event\x00")
|
|
609
|
+
digest.update(f"{event}:{group_index}:{handler_index}".encode("utf-8"))
|
|
610
|
+
for token in tokens:
|
|
611
|
+
digest.update(b"\x00arg\x00")
|
|
612
|
+
digest.update(token.encode("utf-8"))
|
|
613
|
+
return digest.hexdigest()
|
|
614
|
+
|
|
615
|
+
|
|
538
616
|
def codex_hook_roots_all_enabled(roots) -> bool:
|
|
539
617
|
"""Fail-closed frontier predicate: at least one root, every one enabled."""
|
|
540
618
|
roots = list(roots)
|
|
@@ -546,6 +624,30 @@ def codex_hook_roots_all_enabled(roots) -> bool:
|
|
|
546
624
|
return False
|
|
547
625
|
|
|
548
626
|
|
|
627
|
+
def codex_configuration_generation_for_guard_paths(guard_paths) -> "str | None":
|
|
628
|
+
""":func:`codex_configuration_generation` for the roots a guard set names.
|
|
629
|
+
|
|
630
|
+
The inverse of :func:`codex_frontier_guard_paths`, and it lives here beside
|
|
631
|
+
it so the two stay one statement rather than an assumption held in the
|
|
632
|
+
frontier. Every guard set that function produces is, per root, the root's
|
|
633
|
+
``hooks.json`` followed by its ``config.toml``, and a root's home is that
|
|
634
|
+
``hooks.json``'s parent — so the guard set the frontier already holds names
|
|
635
|
+
exactly the roots whose configuration generation it needs, and it needs no
|
|
636
|
+
second channel to be told them.
|
|
637
|
+
|
|
638
|
+
Anything that is not a ``hooks.json`` is ignored rather than rejected: the
|
|
639
|
+
Claude frontier passes its settings path through the same parameter.
|
|
640
|
+
"""
|
|
641
|
+
homes = [
|
|
642
|
+
pathlib.Path(path).parent
|
|
643
|
+
for path in guard_paths
|
|
644
|
+
if pathlib.Path(path).name == "hooks.json"
|
|
645
|
+
]
|
|
646
|
+
if not homes:
|
|
647
|
+
return None
|
|
648
|
+
return codex_configuration_generation(codex_hook_roots(homes))
|
|
649
|
+
|
|
650
|
+
|
|
549
651
|
def codex_frontier_guard_paths(roots) -> tuple[pathlib.Path, ...]:
|
|
550
652
|
"""Every file whose change must retire a Codex frontier certificate.
|
|
551
653
|
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Decode-tolerant Codex thread project metadata (#845).
|
|
2
|
+
|
|
3
|
+
Codex persists a conversation's working directory and git context as
|
|
4
|
+
``codex_conversation_threads.cwd`` and ``.git_json``, both TEXT. Python's
|
|
5
|
+
``sqlite3`` decodes TEXT with the default ``str`` factory and raises
|
|
6
|
+
``OperationalError: Could not decode to UTF-8`` on any statement that selects
|
|
7
|
+
the column, so one undecodable byte failed every reader that touched the two
|
|
8
|
+
columns — whatever the row's age, and whatever else the reader was doing.
|
|
9
|
+
|
|
10
|
+
Every in-scope statement therefore selects ``CAST(t.cwd AS BLOB) AS cwd_blob``
|
|
11
|
+
and ``CAST(t.git_json AS BLOB) AS git_json_blob`` and passes both through
|
|
12
|
+
``decode_codex_project_metadata``. The columns are named ``cwd_blob`` and
|
|
13
|
+
``git_json_blob`` so a reader that forgets to decode fails with a ``KeyError``
|
|
14
|
+
instead of silently receiving bytes. The shared connection's ``text_factory``
|
|
15
|
+
is never changed: it is process-wide state a concurrent reader also depends on.
|
|
16
|
+
|
|
17
|
+
This module is a stdlib-only leaf. It imports nothing from the rest of the
|
|
18
|
+
package, so the qualified accounting reader, the dashboard conversation map,
|
|
19
|
+
the two cache-side rollup writers and the Conversation Viewer can all call one
|
|
20
|
+
implementation of the rule without an import cycle.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import dataclasses
|
|
25
|
+
import sqlite3
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
#: The physical name of the raw threads table. ``resolve_codex_threads_table``
|
|
29
|
+
#: returns it qualified by whichever schema actually holds it.
|
|
30
|
+
CODEX_THREADS_TABLE = "codex_conversation_threads"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclasses.dataclass(frozen=True)
|
|
34
|
+
class DecodedCodexProjectMetadata:
|
|
35
|
+
"""One thread row's two project-metadata fields, decoded field by field.
|
|
36
|
+
|
|
37
|
+
``undecodable_fields`` names the fields whose stored bytes are not valid
|
|
38
|
+
UTF-8. Such a field is present in the store and unreadable here, which is
|
|
39
|
+
a third state beside "absent" and "usable": its own value is ``None``, so a
|
|
40
|
+
caller that ignores the set degrades it to absent, and the malformed
|
|
41
|
+
predicate below is what tells the two apart.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
cwd: str | None
|
|
45
|
+
git_json: str | None
|
|
46
|
+
undecodable_fields: frozenset[str]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _decode_field(value: "bytes | str | None", name: str) -> "tuple[str | None, bool]":
|
|
50
|
+
"""Decode one stored field; report whether its bytes are undecodable."""
|
|
51
|
+
if value is None:
|
|
52
|
+
return None, False
|
|
53
|
+
if isinstance(value, str):
|
|
54
|
+
# A caller that already holds a decoded value (a fixture builder, or a
|
|
55
|
+
# store whose text_factory a test changed) passes it verbatim.
|
|
56
|
+
return value, False
|
|
57
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
58
|
+
try:
|
|
59
|
+
return bytes(value).decode("utf-8"), False
|
|
60
|
+
except UnicodeDecodeError:
|
|
61
|
+
return None, True
|
|
62
|
+
raise TypeError(
|
|
63
|
+
f"{name} must be bytes, str or None, not {type(value).__name__}; "
|
|
64
|
+
"an in-scope statement selects the column as CAST(... AS BLOB)"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def decode_codex_project_metadata(
|
|
69
|
+
cwd_blob: "bytes | str | None", git_json_blob: "bytes | str | None",
|
|
70
|
+
) -> DecodedCodexProjectMetadata:
|
|
71
|
+
"""Decode one thread row's ``cwd`` and ``git_json`` independently.
|
|
72
|
+
|
|
73
|
+
``bytes`` is decoded with strict UTF-8; a failure records the field name in
|
|
74
|
+
``undecodable_fields`` and leaves that field ``None``. ``str`` is accepted
|
|
75
|
+
verbatim and ``None`` stays ``None``. The two fields never share a verdict:
|
|
76
|
+
a row whose ``git_json`` is corrupt but whose ``cwd`` reads fine is still
|
|
77
|
+
attributable, because the resolver stops at a usable ``cwd``.
|
|
78
|
+
"""
|
|
79
|
+
cwd, cwd_bad = _decode_field(cwd_blob, "cwd")
|
|
80
|
+
git_json, git_bad = _decode_field(git_json_blob, "git_json")
|
|
81
|
+
undecodable = set()
|
|
82
|
+
if cwd_bad:
|
|
83
|
+
undecodable.add("cwd")
|
|
84
|
+
if git_bad:
|
|
85
|
+
undecodable.add("git_json")
|
|
86
|
+
return DecodedCodexProjectMetadata(
|
|
87
|
+
cwd=cwd, git_json=git_json, undecodable_fields=frozenset(undecodable),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _absent(metadata: "DecodedCodexProjectMetadata | None", field: str) -> bool:
|
|
92
|
+
"""Whether the resolver would move past this field.
|
|
93
|
+
|
|
94
|
+
"Absent" means the row is missing, or the field is NULL or empty — exactly
|
|
95
|
+
the values ``cwd = direct.cwd or inherited.cwd`` skips. An undecodable
|
|
96
|
+
field is absent by this test too, which is why the caller checks
|
|
97
|
+
``_undecodable`` first.
|
|
98
|
+
"""
|
|
99
|
+
if metadata is None:
|
|
100
|
+
return True
|
|
101
|
+
return not getattr(metadata, field)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _undecodable(metadata: "DecodedCodexProjectMetadata | None", field: str) -> bool:
|
|
105
|
+
return metadata is not None and field in metadata.undecodable_fields
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def codex_metadata_is_malformed(
|
|
109
|
+
direct: "DecodedCodexProjectMetadata | None",
|
|
110
|
+
inherited: "DecodedCodexProjectMetadata | None",
|
|
111
|
+
) -> bool:
|
|
112
|
+
"""Whether the resolver's path reaches an undecodable field (§4.1).
|
|
113
|
+
|
|
114
|
+
``_qualify_codex_rows`` merges each field independently, ``cwd =
|
|
115
|
+
direct.cwd or inherited.cwd`` and ``git_json = direct.git_json or
|
|
116
|
+
inherited.git_json``, and ``_CodexProjectResolution.resolve`` stops at a
|
|
117
|
+
usable merged ``cwd`` and consults ``git_json`` only otherwise. A field is
|
|
118
|
+
"consulted" when the resolver's outcome depends on it, and an undecodable
|
|
119
|
+
field is present but unreadable, so whenever the path reaches one the entry
|
|
120
|
+
cannot be attributed:
|
|
121
|
+
|
|
122
|
+
- **E1.** Direct ``cwd`` undecodable.
|
|
123
|
+
- **E2.** Direct ``cwd`` absent and inherited ``cwd`` undecodable.
|
|
124
|
+
- **E3.** Direct and inherited ``cwd`` absent, direct ``git_json``
|
|
125
|
+
undecodable.
|
|
126
|
+
- **E4.** Direct and inherited ``cwd`` absent, direct ``git_json`` absent,
|
|
127
|
+
inherited ``git_json`` undecodable.
|
|
128
|
+
|
|
129
|
+
Otherwise the entry is attributable, which is the half that keeps healthy
|
|
130
|
+
rows healthy: a valid direct ``cwd`` beside an undecodable direct
|
|
131
|
+
``git_json`` still resolves, and so does a valid inherited ``cwd`` beside
|
|
132
|
+
an undecodable inherited ``git_json``.
|
|
133
|
+
|
|
134
|
+
The qualified reader raises for a malformed entry, the counter counts it,
|
|
135
|
+
the identity map withholds its attribution and the viewer's key set admits
|
|
136
|
+
it, and all four call THIS function with the entry's direct row and its
|
|
137
|
+
path's inherited winner, so the refusal and the count cannot disagree.
|
|
138
|
+
"""
|
|
139
|
+
if _undecodable(direct, "cwd"):
|
|
140
|
+
return True
|
|
141
|
+
if not _absent(direct, "cwd"):
|
|
142
|
+
return False
|
|
143
|
+
if _undecodable(inherited, "cwd"):
|
|
144
|
+
return True
|
|
145
|
+
if not _absent(inherited, "cwd"):
|
|
146
|
+
return False
|
|
147
|
+
if _undecodable(direct, "git_json"):
|
|
148
|
+
return True
|
|
149
|
+
if not _absent(direct, "git_json"):
|
|
150
|
+
return False
|
|
151
|
+
return _undecodable(inherited, "git_json")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _schema_holds_threads_table(conn: sqlite3.Connection, schema: str) -> bool:
|
|
155
|
+
"""Whether ``schema`` holds the threads table; a read failure propagates.
|
|
156
|
+
|
|
157
|
+
``main`` always exists and ``cache_db`` is probed only after
|
|
158
|
+
``PRAGMA database_list`` named it, so the only ``sqlite3.Error`` this
|
|
159
|
+
statement can raise is a store the connection cannot read (locked,
|
|
160
|
+
malformed). Swallowing it here answered ``False``, which the resolver
|
|
161
|
+
turned into its absent-table ``RuntimeError`` and the two fail-closed
|
|
162
|
+
reads (#850 M5) then answered with the healthy value; a locked store hits
|
|
163
|
+
this probe before it hits their own statement.
|
|
164
|
+
"""
|
|
165
|
+
row = conn.execute(
|
|
166
|
+
f'SELECT 1 FROM "{schema}".sqlite_master '
|
|
167
|
+
"WHERE type = 'table' AND name = ?",
|
|
168
|
+
(CODEX_THREADS_TABLE,),
|
|
169
|
+
).fetchone()
|
|
170
|
+
return row is not None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def resolve_codex_threads_table(conn: sqlite3.Connection) -> str:
|
|
174
|
+
"""The RAW threads table reference for this connection, resolved at call time.
|
|
175
|
+
|
|
176
|
+
A bare cache connection (``open_cache_db()``) holds the table in its own
|
|
177
|
+
``main`` schema; a conversations connection
|
|
178
|
+
(``open_conversations_db``/``open_conversations_db_readonly``) reaches it
|
|
179
|
+
through the cache attached as ``cache_db``. A qualified reference resolves
|
|
180
|
+
to the real table even when account scoping has installed an empty TEMP
|
|
181
|
+
view over the unqualified name, which is what lets the fail-closed
|
|
182
|
+
anonymization count and the viewer's key set be observed by a scoped
|
|
183
|
+
request too.
|
|
184
|
+
|
|
185
|
+
Readers that must HONOR scoping keep the unqualified name; this is only for
|
|
186
|
+
readers that must see the raw rows. A hard-coded ``cache_db.`` prefix fails
|
|
187
|
+
with ``no such table`` on the bare cache connection, which is why the
|
|
188
|
+
reference is resolved rather than written out.
|
|
189
|
+
|
|
190
|
+
``RuntimeError`` means exactly one thing: the table is absent from every
|
|
191
|
+
schema this connection can see. A ``sqlite3.Error`` raised while
|
|
192
|
+
resolving is a read the store could not perform and propagates unchanged,
|
|
193
|
+
so a caller that answers the absent-table case with a healthy value
|
|
194
|
+
(zero undecodable rows, the empty key set) never gives that answer for a
|
|
195
|
+
locked or malformed store.
|
|
196
|
+
"""
|
|
197
|
+
if _schema_holds_threads_table(conn, "main"):
|
|
198
|
+
return f"main.{CODEX_THREADS_TABLE}"
|
|
199
|
+
attached = tuple(conn.execute("PRAGMA database_list"))
|
|
200
|
+
for row in attached:
|
|
201
|
+
name = str(row[1]) if len(row) > 1 else ""
|
|
202
|
+
if name == "cache_db" and _schema_holds_threads_table(conn, name):
|
|
203
|
+
return f"cache_db.{CODEX_THREADS_TABLE}"
|
|
204
|
+
raise RuntimeError(
|
|
205
|
+
"the Codex threads table is not present in this connection's main "
|
|
206
|
+
"schema and no cache database is attached as cache_db"
|
|
207
|
+
)
|
package/bin/_lib_conversation.py
CHANGED
|
@@ -1415,3 +1415,122 @@ def iter_media_items(content):
|
|
|
1415
1415
|
if isinstance(item, dict) and item.get("type") in _MEDIA_BLOCK_TYPES:
|
|
1416
1416
|
yield idx, item
|
|
1417
1417
|
idx += 1
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
# --- #777 source-incarnation identity and durable account stamps -----------
|
|
1421
|
+
# ONE definition of each, shared by the migration-009 backfill in
|
|
1422
|
+
# `_cctally_db` and the ingest boundary in `_cctally_cache`. Two definitions
|
|
1423
|
+
# would not raise, would not move a golden, and would not fail a suite: the
|
|
1424
|
+
# migration's stamps would simply never match the ingester's lookups, and every
|
|
1425
|
+
# rebuild after the first would silently re-attribute history to whichever
|
|
1426
|
+
# account happened to be active. That is the failure mode this module's
|
|
1427
|
+
# single-kernel rule exists to prevent.
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def record_sha256(raw_span: bytes) -> str:
|
|
1431
|
+
"""The digest of ONE record's raw on-disk bytes.
|
|
1432
|
+
|
|
1433
|
+
`raw_span` is the byte span of the record as it appears on disk, with the
|
|
1434
|
+
terminator already removed. Digesting raw bytes rather than decoded text is
|
|
1435
|
+
load-bearing: the sync walker reads with `errors="replace"`, so a record
|
|
1436
|
+
carrying invalid UTF-8 decodes to a DIFFERENT string than it was written
|
|
1437
|
+
as, and a digest over that string would change the moment the decoder's
|
|
1438
|
+
replacement policy did. Re-serializing the parsed object would be worse
|
|
1439
|
+
still — key order and separator choices are not stable across versions.
|
|
1440
|
+
"""
|
|
1441
|
+
import hashlib # noqa: PLC0415 — stdlib, kept off this module's import cost
|
|
1442
|
+
return hashlib.sha256(raw_span).hexdigest()
|
|
1443
|
+
|
|
1444
|
+
|
|
1445
|
+
def strip_record_terminator(line: bytes) -> bytes:
|
|
1446
|
+
"""The record's bytes with its line terminator removed.
|
|
1447
|
+
|
|
1448
|
+
`\\r\\n` is removed WHOLE, so the same record content digests identically
|
|
1449
|
+
whichever terminator wrote it. That is a deliberate reading of "excluding
|
|
1450
|
+
the terminating newline": treating the `\\r` as record content would make
|
|
1451
|
+
one logical record produce two identities across a file rewritten on
|
|
1452
|
+
another platform, and every stamp for it would miss.
|
|
1453
|
+
"""
|
|
1454
|
+
if line.endswith(b"\n"):
|
|
1455
|
+
line = line[:-1]
|
|
1456
|
+
if line.endswith(b"\r"):
|
|
1457
|
+
line = line[:-1]
|
|
1458
|
+
return line
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def new_source_incarnation_id() -> str:
|
|
1462
|
+
"""Mint an identifier for one append-continuous life of a source file.
|
|
1463
|
+
|
|
1464
|
+
Random rather than derived from the file's content or metadata. A derived
|
|
1465
|
+
id would collide exactly when the file is rewritten to look like itself —
|
|
1466
|
+
the size-preserving in-place rewrite this identity exists to detect — and a
|
|
1467
|
+
colliding id lets a new incarnation inherit the old one's stamps.
|
|
1468
|
+
"""
|
|
1469
|
+
import uuid # noqa: PLC0415 — stdlib, kept off this module's import cost
|
|
1470
|
+
return uuid.uuid4().hex
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
def prefix_digest(fh, length: int):
|
|
1474
|
+
"""A live SHA-256 object over `[0, length)`, read through an ALREADY-OPEN
|
|
1475
|
+
binary handle.
|
|
1476
|
+
|
|
1477
|
+
Returns the hash OBJECT rather than a hex string, because the ingester
|
|
1478
|
+
continues it over the bytes it then appends. A hex digest cannot be
|
|
1479
|
+
resumed, so returning one would force a full rehash of the whole file on
|
|
1480
|
+
every sync instead of only the committed prefix.
|
|
1481
|
+
|
|
1482
|
+
A finite window ending at the cursor cannot keep the continuity promise,
|
|
1483
|
+
because a rewrite entirely before that window leaves the window unchanged.
|
|
1484
|
+
The guard is therefore the whole committed prefix. Verification on resume
|
|
1485
|
+
rehashes it once, which is bounded by the file rather than by the corpus,
|
|
1486
|
+
and only for files whose size actually changed — an unchanged file is
|
|
1487
|
+
skipped before this is reached.
|
|
1488
|
+
|
|
1489
|
+
Reads through the caller's descriptor rather than re-opening by path, so
|
|
1490
|
+
the digest and the caller's `fstat` pair describe the same open file.
|
|
1491
|
+
|
|
1492
|
+
POSITION CONTRACT. On return the handle is positioned at `length` — or at
|
|
1493
|
+
end of file when the file is shorter. The ingester relies on that: it
|
|
1494
|
+
continues this same object over the bytes it then reads, so a caller that
|
|
1495
|
+
re-seeks between the two would digest the wrong span. State it here rather
|
|
1496
|
+
than leaving the caller to depend on it silently.
|
|
1497
|
+
|
|
1498
|
+
A SHORT READ RAISES. Reading fewer than `length` bytes means the file no
|
|
1499
|
+
longer holds the prefix the cursor describes, and returning a digest over
|
|
1500
|
+
what was there would produce a hash of a different byte range wearing the
|
|
1501
|
+
name of the committed prefix — which then compares unequal for a reason the
|
|
1502
|
+
caller cannot see. `ValueError` says so directly; the caller's incarnation
|
|
1503
|
+
resolver treats it as a broken continuity, which is the correct answer.
|
|
1504
|
+
"""
|
|
1505
|
+
import hashlib # noqa: PLC0415 — stdlib, kept off this module's import cost
|
|
1506
|
+
digest = hashlib.sha256()
|
|
1507
|
+
if length <= 0:
|
|
1508
|
+
return digest
|
|
1509
|
+
fh.seek(0)
|
|
1510
|
+
remaining = length
|
|
1511
|
+
while remaining > 0:
|
|
1512
|
+
chunk = fh.read(min(remaining, 1 << 20))
|
|
1513
|
+
if not chunk:
|
|
1514
|
+
raise ValueError(
|
|
1515
|
+
f"prefix_digest: read {length - remaining} of {length} bytes; "
|
|
1516
|
+
"the file no longer holds the committed prefix"
|
|
1517
|
+
)
|
|
1518
|
+
digest.update(chunk)
|
|
1519
|
+
remaining -= len(chunk)
|
|
1520
|
+
return digest
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
def prefix_sha256(fh, length: int) -> str:
|
|
1524
|
+
"""SHA-256 of `[0, length)` read through an ALREADY-OPEN binary handle.
|
|
1525
|
+
|
|
1526
|
+
A finite window ending at the cursor cannot keep the continuity promise,
|
|
1527
|
+
because a rewrite entirely before that window leaves the window unchanged.
|
|
1528
|
+
The guard is therefore the whole committed prefix. Verification on resume
|
|
1529
|
+
rehashes it once, which is bounded by the file rather than by the corpus,
|
|
1530
|
+
and only for files whose size actually changed — an unchanged file is
|
|
1531
|
+
skipped before this is reached.
|
|
1532
|
+
|
|
1533
|
+
Reads through the caller's descriptor rather than re-opening by path, so
|
|
1534
|
+
the digest and the caller's `fstat` pair describe the same open file.
|
|
1535
|
+
"""
|
|
1536
|
+
return prefix_digest(fh, length).hexdigest()
|
|
@@ -290,6 +290,29 @@ def _claude_browse(
|
|
|
290
290
|
for conv in convs
|
|
291
291
|
]
|
|
292
292
|
facets = q._browse_facets(rows)
|
|
293
|
+
# #717. Computed HERE, at the source. An earlier draft of the spec claimed
|
|
294
|
+
# this function already computed a `filter_degraded` and merely discarded
|
|
295
|
+
# it; it did not, and correcting that is why §6 asks for computation
|
|
296
|
+
# rather than propagation. The authority is `_rollup_authoritative` — the
|
|
297
|
+
# absence of `conversation_sessions_backfill_pending` — and never the
|
|
298
|
+
# pricing refusal latch, which coexists with an intact rollup.
|
|
299
|
+
#
|
|
300
|
+
# DELIBERATE DIVERGENCE from §6's "when degraded, projects: []", for this
|
|
301
|
+
# producer only. §6 states that rule once for both Claude paths, and it is
|
|
302
|
+
# right for the unqualified one, whose `projects` is a GROUP BY over the
|
|
303
|
+
# `conversation_sessions` table and really is empty while that table is
|
|
304
|
+
# being rebuilt. It is wrong here: these facets are derived from the browse
|
|
305
|
+
# ROWS, and `_all_claude_conversations` pages `lcq.list_conversations`,
|
|
306
|
+
# which falls back to a live GROUP BY over `conversation_messages` whenever
|
|
307
|
+
# the rollup is not authoritative, so the rows — and the project facet
|
|
308
|
+
# built from them — stay complete. Emptying them anyway hid two real
|
|
309
|
+
# projects behind a "still indexing" message, which is what
|
|
310
|
+
# `test_neutral_browse_claude_same_basename_distinct_project_key` caught.
|
|
311
|
+
#
|
|
312
|
+
# The flag therefore reports the same fact without the false emptiness, and
|
|
313
|
+
# the client renders the explanatory line only where the list IS empty, so
|
|
314
|
+
# a complete list still renders as itself.
|
|
315
|
+
degraded = not lcq._rollup_authoritative(conn)
|
|
293
316
|
filtered = [
|
|
294
317
|
row for row in rows
|
|
295
318
|
if (project_key is None or row["project_key"] == project_key)
|
|
@@ -298,6 +321,8 @@ def _claude_browse(
|
|
|
298
321
|
filtered.sort(key=q._recent_sort_key, reverse=True)
|
|
299
322
|
page_rows, page = q._paginate_rows(filtered, cursor=cursor, limit=limit)
|
|
300
323
|
result = {"status": "ok", "rows": page_rows, "facets": facets, "page": page}
|
|
324
|
+
if degraded:
|
|
325
|
+
result["filter_degraded"] = True
|
|
301
326
|
if selected is not None:
|
|
302
327
|
selected_row = next(
|
|
303
328
|
(row for row in rows if row["conversation_key"] == selected), None)
|
|
@@ -655,8 +680,13 @@ def neutral_facets(
|
|
|
655
680
|
return q.list_codex_conversation_facets(conn)
|
|
656
681
|
if source == "claude":
|
|
657
682
|
env = _claude_browse(conn, effective_speed=speed)
|
|
658
|
-
|
|
683
|
+
# #717: this reshaping site keeps only `status` and `facets`, so the
|
|
684
|
+
# flag has to be carried across it explicitly or it is dropped here.
|
|
685
|
+
out = {"status": env.get("status"), "facets": env.get("facets") or {
|
|
659
686
|
"projects": [], "models": []}}
|
|
687
|
+
if env.get("filter_degraded"):
|
|
688
|
+
out["filter_degraded"] = True
|
|
689
|
+
return out
|
|
660
690
|
raise ValueError(f"unknown source: {source!r}")
|
|
661
691
|
|
|
662
692
|
|