cctally 1.114.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4491 -295
- package/bin/_cctally_completion.py +1355 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +282 -9
- package/bin/_cctally_dashboard.py +8 -11
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +42 -35
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +401 -9
- package/bin/_cctally_doctor.py +469 -27
- package/bin/_cctally_five_hour.py +166 -1049
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +5076 -177
- package/bin/_cctally_journal_repair.py +110 -19
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +47 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +662 -0
- package/bin/_cctally_quota.py +96 -12
- package/bin/_cctally_quota_model.py +391 -83
- package/bin/_cctally_record.py +1088 -269
- package/bin/_cctally_rederive.py +854 -192
- package/bin/_cctally_refresh.py +113 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +7062 -505
- package/bin/_cctally_store.py +1244 -91
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +119 -60
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +569 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +177 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +4038 -0
- package/bin/_lib_doctor.py +753 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +261 -10
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +155 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +384 -52
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +393 -8
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +356 -27
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +29 -12
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +20 -2
- package/bin/_lib_merge_gate.py +0 -725
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Clone-first file and tree copies (stdlib only; #940 D2, spec §6.7).
|
|
2
|
+
|
|
3
|
+
On APFS, ``fclonefileat(2)`` gives a copy that shares the source's blocks
|
|
4
|
+
until either side is written, so a copy of a large test scaffold writes no
|
|
5
|
+
data. This module takes that clone wherever it can and a real copy wherever
|
|
6
|
+
it cannot, and otherwise behaves exactly as the stdlib copier it stands in
|
|
7
|
+
for:
|
|
8
|
+
|
|
9
|
+
- :func:`copy2` is ``shutil.copy2`` for the same arguments: a directory
|
|
10
|
+
destination keeps the source's basename, ``follow_symlinks=False`` copies a
|
|
11
|
+
link as a link, the same file raises ``shutil.SameFileError``, and an
|
|
12
|
+
existing destination is overwritten in place (same inode), which a clone
|
|
13
|
+
cannot do, so that case is a real copy. Metadata is ``shutil.copystat``'s,
|
|
14
|
+
applied after the clone as after a copy.
|
|
15
|
+
- :func:`try_clone` is the clone alone: True when `dst` was created as a
|
|
16
|
+
clone, False (nothing created) when the platform or filesystem refuses, so
|
|
17
|
+
a caller can choose where its real copy goes (#997 D13).
|
|
18
|
+
- :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
|
|
19
|
+
copier, so symlink policy, ignore lists, dangling-link errors and
|
|
20
|
+
destination collisions are the stdlib's own code.
|
|
21
|
+
|
|
22
|
+
A clone is attempted only on Darwin. An unsupported filesystem, a
|
|
23
|
+
cross-volume destination, a missing ``fclonefileat`` and any non-Darwin
|
|
24
|
+
platform take the real copy; any other error (EIO, ENOSPC, EACCES, …)
|
|
25
|
+
propagates and is never retried as a copy. A clone is never a hardlink: the
|
|
26
|
+
copy is a new inode with a link count of one.
|
|
27
|
+
|
|
28
|
+
The optional ``counts`` mapping is incremented under ``"clone"`` or
|
|
29
|
+
``"copy"`` per file, so a caller or a measurement can see which path ran.
|
|
30
|
+
This module is public and imports nothing outside the standard library.
|
|
31
|
+
|
|
32
|
+
It binds ``fclonefileat`` itself rather than importing the binding from the
|
|
33
|
+
retired private staging module (#940 D-2 F17): a private module is absent
|
|
34
|
+
from the public tree that this module and its public callers ship in, and a
|
|
35
|
+
public module may not import it.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
import ctypes
|
|
41
|
+
import errno
|
|
42
|
+
import os
|
|
43
|
+
import shutil
|
|
44
|
+
import stat
|
|
45
|
+
import sys
|
|
46
|
+
|
|
47
|
+
# Errors that mean "this filesystem or this pair of volumes cannot clone",
|
|
48
|
+
# never "the data could not be written". EEXIST is a destination created
|
|
49
|
+
# after the existence check: `shutil.copy2` overwrites it, and so do we.
|
|
50
|
+
_FALLBACK_ERRNOS = frozenset(
|
|
51
|
+
code for code in (
|
|
52
|
+
getattr(errno, "ENOTSUP", None), getattr(errno, "EOPNOTSUPP", None),
|
|
53
|
+
errno.ENOSYS, errno.EXDEV, errno.EINVAL, errno.EPERM, errno.EEXIST)
|
|
54
|
+
if code is not None)
|
|
55
|
+
|
|
56
|
+
_AT_FDCWD = -2 # Darwin's AT_FDCWD
|
|
57
|
+
_CLONE_NOOWNERCOPY = 0x0002 # leave ownership to the caller, as a copy does
|
|
58
|
+
|
|
59
|
+
_LIBC_CLONE = [] # the resolved fclonefileat, once looked up
|
|
60
|
+
# The platform this module decides on; a test sets it here instead of
|
|
61
|
+
# patching the interpreter-wide ``sys.platform`` (#940 D-2 F16).
|
|
62
|
+
_PLATFORM = sys.platform
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _fclonefileat():
|
|
66
|
+
"""The C ``fclonefileat`` on Darwin, else None."""
|
|
67
|
+
if _PLATFORM != "darwin":
|
|
68
|
+
return None
|
|
69
|
+
if not _LIBC_CLONE:
|
|
70
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
71
|
+
clone = getattr(libc, "fclonefileat", None)
|
|
72
|
+
if clone is not None:
|
|
73
|
+
clone.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32)
|
|
74
|
+
clone.restype = ctypes.c_int
|
|
75
|
+
_LIBC_CLONE.append(clone)
|
|
76
|
+
return _LIBC_CLONE[0]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _clone_fd(source_fd: int, destination: str) -> None:
|
|
80
|
+
"""Clone the open regular file `source_fd` to the new path `destination`."""
|
|
81
|
+
clone = _fclonefileat()
|
|
82
|
+
if clone is None:
|
|
83
|
+
raise OSError(errno.ENOTSUP, "fclonefileat is unavailable", destination)
|
|
84
|
+
if clone(source_fd, _AT_FDCWD, os.fsencode(os.path.abspath(destination)),
|
|
85
|
+
_CLONE_NOOWNERCOPY) != 0:
|
|
86
|
+
code = ctypes.get_errno()
|
|
87
|
+
raise OSError(code, os.strerror(code), destination)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _clone_open_file(fd: int, src, dst, *, follow_symlinks: bool = True) -> bool:
|
|
91
|
+
"""Clone the open regular file `fd` (opened from `src`) to the new path
|
|
92
|
+
`dst` and apply `src`'s metadata. False, with nothing created, when the
|
|
93
|
+
filesystem or the pair of volumes refuses the clone (`_FALLBACK_ERRNOS`);
|
|
94
|
+
any other error propagates."""
|
|
95
|
+
try:
|
|
96
|
+
_clone_fd(fd, dst)
|
|
97
|
+
except OSError as exc:
|
|
98
|
+
if exc.errno not in _FALLBACK_ERRNOS:
|
|
99
|
+
raise
|
|
100
|
+
return False
|
|
101
|
+
shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
|
|
102
|
+
return True
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def try_clone(src, dst) -> bool:
|
|
106
|
+
"""Clone the regular file `src` to the NEW path `dst` (#997 D13).
|
|
107
|
+
|
|
108
|
+
True: `dst` is a clone sharing `src`'s blocks, with ``shutil.copystat``
|
|
109
|
+
metadata applied. False, with `dst` not created: the platform is not
|
|
110
|
+
Darwin (or has no ``fclonefileat``), `src` is not a regular file, or the
|
|
111
|
+
filesystem refused with a fallback errno (an unsupported filesystem, a
|
|
112
|
+
cross-volume destination, an existing `dst`). Any other error, including
|
|
113
|
+
a `src` that cannot be opened, propagates. A caller that needs a copy
|
|
114
|
+
either way makes the real copy itself on False.
|
|
115
|
+
"""
|
|
116
|
+
if _fclonefileat() is None:
|
|
117
|
+
return False
|
|
118
|
+
fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
|
|
119
|
+
try:
|
|
120
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
121
|
+
return False
|
|
122
|
+
return _clone_open_file(fd, src, dst)
|
|
123
|
+
finally:
|
|
124
|
+
os.close(fd)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _count(counts, method: str) -> None:
|
|
128
|
+
if counts is not None:
|
|
129
|
+
counts[method] = counts.get(method, 0) + 1
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def copy2(src, dst, *, follow_symlinks: bool = True, counts=None):
|
|
133
|
+
"""``shutil.copy2(src, dst)``, by clone where the filesystem allows it.
|
|
134
|
+
|
|
135
|
+
Returns the destination path, as ``shutil.copy2`` does.
|
|
136
|
+
"""
|
|
137
|
+
if os.path.isdir(dst):
|
|
138
|
+
dst = os.path.join(dst, os.path.basename(src))
|
|
139
|
+
if (not follow_symlinks and os.path.islink(src)) or os.path.lexists(dst):
|
|
140
|
+
# A link copied as a link, or a destination to overwrite in place.
|
|
141
|
+
result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
142
|
+
_count(counts, "copy")
|
|
143
|
+
return result
|
|
144
|
+
try:
|
|
145
|
+
fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
|
|
146
|
+
except OSError:
|
|
147
|
+
# The stdlib's own error for this source (missing, unreadable, …).
|
|
148
|
+
return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
149
|
+
try:
|
|
150
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
151
|
+
# A FIFO, a device or a directory: the stdlib decides.
|
|
152
|
+
os.close(fd)
|
|
153
|
+
fd = -1
|
|
154
|
+
return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
155
|
+
if not _clone_open_file(fd, src, dst,
|
|
156
|
+
follow_symlinks=follow_symlinks):
|
|
157
|
+
result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
158
|
+
_count(counts, "copy")
|
|
159
|
+
return result
|
|
160
|
+
_count(counts, "clone")
|
|
161
|
+
return dst
|
|
162
|
+
finally:
|
|
163
|
+
if fd >= 0:
|
|
164
|
+
os.close(fd)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def copytree(src, dst, symlinks: bool = False, ignore=None,
|
|
168
|
+
ignore_dangling_symlinks: bool = False, dirs_exist_ok: bool = False,
|
|
169
|
+
*, counts=None):
|
|
170
|
+
"""``shutil.copytree`` with :func:`copy2` as its file copier."""
|
|
171
|
+
def copier(source, destination):
|
|
172
|
+
return copy2(source, destination, counts=counts)
|
|
173
|
+
|
|
174
|
+
return shutil.copytree(src, dst, symlinks=symlinks, ignore=ignore,
|
|
175
|
+
copy_function=copier,
|
|
176
|
+
ignore_dangling_symlinks=ignore_dangling_symlinks,
|
|
177
|
+
dirs_exist_ok=dirs_exist_ok)
|
|
@@ -27,11 +27,13 @@ import json
|
|
|
27
27
|
import os
|
|
28
28
|
import re
|
|
29
29
|
import sqlite3
|
|
30
|
+
import stat
|
|
30
31
|
import threading
|
|
31
32
|
from collections import deque
|
|
32
33
|
from types import MappingProxyType
|
|
33
34
|
|
|
34
35
|
import _lib_codex_conversation as kern
|
|
36
|
+
import _lib_codex_rollout_container as _rc
|
|
35
37
|
import _lib_codex_landmarks as landmarks
|
|
36
38
|
import _lib_codex_segments as segkern
|
|
37
39
|
from _lib_codex_find_projection import (
|
|
@@ -1863,6 +1865,29 @@ def codex_conversation_exists(conn: sqlite3.Connection, conversation_key: str) -
|
|
|
1863
1865
|
return row is not None
|
|
1864
1866
|
|
|
1865
1867
|
|
|
1868
|
+
def codex_publication_pending(
|
|
1869
|
+
conn: sqlite3.Connection, conversation_key: str,
|
|
1870
|
+
) -> bool:
|
|
1871
|
+
"""A barrier-population conversation with no publication yet (#968 5.11).
|
|
1872
|
+
|
|
1873
|
+
Such a conversation stays in the existing pending-state handling until it
|
|
1874
|
+
has an eligible publication: it answers ``normalization_pending``, never
|
|
1875
|
+
``not_found``. Any store error answers False (the caller's ``not_found``).
|
|
1876
|
+
"""
|
|
1877
|
+
try:
|
|
1878
|
+
if conn.execute(
|
|
1879
|
+
"SELECT 1 FROM cache_db.codex_barrier_population "
|
|
1880
|
+
"WHERE conversation_key=?", (conversation_key,)).fetchone() is None:
|
|
1881
|
+
return False
|
|
1882
|
+
row = conn.execute(
|
|
1883
|
+
"SELECT published_revision, legacy_uncertified "
|
|
1884
|
+
"FROM main.codex_conversation_publications WHERE conversation_key=?",
|
|
1885
|
+
(conversation_key,)).fetchone()
|
|
1886
|
+
except sqlite3.Error:
|
|
1887
|
+
return False
|
|
1888
|
+
return row is None or (int(row[0] or 0) == 0 and not row[1])
|
|
1889
|
+
|
|
1890
|
+
|
|
1866
1891
|
def codex_conversation_source_paths(
|
|
1867
1892
|
conn: sqlite3.Connection, conversation_key: str
|
|
1868
1893
|
) -> list[str]:
|
|
@@ -2345,6 +2370,9 @@ def get_codex_conversation(
|
|
|
2345
2370
|
# ── Phase A: the narrow index pass ───────────────────────────────────────
|
|
2346
2371
|
rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
|
|
2347
2372
|
if not rows:
|
|
2373
|
+
if codex_publication_pending(conn, conversation_key):
|
|
2374
|
+
return {"status": "normalization_pending",
|
|
2375
|
+
"conversation_key": conversation_key, "items": [], "children": []}
|
|
2348
2376
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
2349
2377
|
undecodable_keys = undecodable_codex_conversation_keys(conn)
|
|
2350
2378
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
@@ -3035,6 +3063,9 @@ def _outline_envelope(
|
|
|
3035
3063
|
# The outline is not the route F3 bounds, and it stays turn-granular.
|
|
3036
3064
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
3037
3065
|
if not rows:
|
|
3066
|
+
if codex_publication_pending(conn, conversation_key):
|
|
3067
|
+
return {"status": "normalization_pending",
|
|
3068
|
+
"conversation_key": conversation_key, "turns": [], "files": [], "children": []}
|
|
3038
3069
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
3039
3070
|
detail_bytes = _detail_bytes_of(rows)
|
|
3040
3071
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
@@ -3421,20 +3452,26 @@ def _stored_browse_page_queries(
|
|
|
3421
3452
|
"(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
|
|
3422
3453
|
page_params.extend((cursor_last, cursor_last, cursor_key))
|
|
3423
3454
|
|
|
3455
|
+
# Keep native/parent enrichment inside the page statement. A LEFT JOIN
|
|
3456
|
+
# against the compound last-good view materializes retained history;
|
|
3457
|
+
# correlated probes push these predicates into its indexed branches.
|
|
3458
|
+
# The scalar parent lookup follows the detail reader's same-root lookup.
|
|
3459
|
+
parent_key_sql = (
|
|
3460
|
+
"(SELECT pt.conversation_key FROM codex_conversation_threads pt "
|
|
3461
|
+
"WHERE pt.source_root_key = r.source_root_key "
|
|
3462
|
+
"AND pt.native_thread_id = r.parent_thread_id "
|
|
3463
|
+
"AND pt.conversation_key != r.conversation_key LIMIT 1)"
|
|
3464
|
+
)
|
|
3424
3465
|
sql = (
|
|
3425
3466
|
"SELECT r.conversation_key, r.item_count, r.started_utc, "
|
|
3426
3467
|
"r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
|
|
3427
|
-
"r.title, r.parent_thread_id, r.source_root_key,
|
|
3428
|
-
"
|
|
3468
|
+
"r.title, r.parent_thread_id, r.source_root_key, "
|
|
3469
|
+
"(SELECT t.native_thread_id FROM codex_conversation_threads t "
|
|
3470
|
+
"WHERE t.conversation_key = r.conversation_key), "
|
|
3471
|
+
f"{parent_key_sql}, pr.title, pr.project_label, r.parent_thread_id "
|
|
3429
3472
|
"FROM codex_conversation_rollups r "
|
|
3430
|
-
"LEFT JOIN codex_conversation_threads t "
|
|
3431
|
-
"ON t.conversation_key = r.conversation_key "
|
|
3432
|
-
"LEFT JOIN codex_conversation_threads pt "
|
|
3433
|
-
"ON pt.source_root_key = r.source_root_key "
|
|
3434
|
-
"AND pt.native_thread_id = r.parent_thread_id "
|
|
3435
|
-
"AND pt.conversation_key != r.conversation_key "
|
|
3436
3473
|
"LEFT JOIN codex_conversation_rollups pr "
|
|
3437
|
-
"ON pr.conversation_key =
|
|
3474
|
+
f"ON pr.conversation_key = {parent_key_sql} "
|
|
3438
3475
|
f"WHERE {' AND '.join(page_where)} "
|
|
3439
3476
|
"ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
|
|
3440
3477
|
)
|
|
@@ -4962,6 +4999,8 @@ def find_in_codex_conversation(
|
|
|
4962
4999
|
return {**base, "status": "normalization_pending"}
|
|
4963
5000
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
4964
5001
|
if not rows:
|
|
5002
|
+
if codex_publication_pending(conn, conversation_key):
|
|
5003
|
+
return {**base, "status": "normalization_pending"}
|
|
4965
5004
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
4966
5005
|
q = (query or "").strip()
|
|
4967
5006
|
if not q:
|
|
@@ -5021,6 +5060,9 @@ def codex_conversation_prompts(conn: sqlite3.Connection, conversation_key: str)
|
|
|
5021
5060
|
"conversation_key": conversation_key, "prompts": []}
|
|
5022
5061
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
5023
5062
|
if not rows:
|
|
5063
|
+
if codex_publication_pending(conn, conversation_key):
|
|
5064
|
+
return {"status": "normalization_pending",
|
|
5065
|
+
"conversation_key": conversation_key, "prompts": []}
|
|
5024
5066
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
5025
5067
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
5026
5068
|
items = kern.canonical_items(kept)
|
|
@@ -5061,7 +5103,40 @@ def _within_root(source_path: str | None, root_path: str | None) -> bool:
|
|
|
5061
5103
|
return False
|
|
5062
5104
|
|
|
5063
5105
|
|
|
5064
|
-
|
|
5106
|
+
class CodexContainerReadError(RuntimeError):
|
|
5107
|
+
"""A present rollout failed to read; the query envelope reports HTTP 500."""
|
|
5108
|
+
|
|
5109
|
+
def __init__(self, category: str, logical: str) -> None:
|
|
5110
|
+
super().__init__(f"Codex rollout could not be read ({category})")
|
|
5111
|
+
self.category = category
|
|
5112
|
+
self.logical = logical
|
|
5113
|
+
|
|
5114
|
+
|
|
5115
|
+
class _CodexPayloadContainmentError(RuntimeError):
|
|
5116
|
+
"""Internal 404 signal: the selected physical container escaped its root."""
|
|
5117
|
+
|
|
5118
|
+
|
|
5119
|
+
def _select_codex_payload_container(logical):
|
|
5120
|
+
# The discovery leaf deliberately treats stat failures as absent. A payload
|
|
5121
|
+
# request must distinguish a missing container (410) from inaccessible data
|
|
5122
|
+
# (500), including an inaccessible plain file beside a readable zstd twin.
|
|
5123
|
+
for candidate in (logical, _rc.zstd_twin(logical)):
|
|
5124
|
+
try:
|
|
5125
|
+
info = os.stat(candidate)
|
|
5126
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
5127
|
+
continue
|
|
5128
|
+
except OSError as exc:
|
|
5129
|
+
raise CodexContainerReadError("read-error", str(logical)) from exc
|
|
5130
|
+
if not stat.S_ISREG(info.st_mode):
|
|
5131
|
+
raise CodexContainerReadError("read-error", str(logical))
|
|
5132
|
+
selected = _rc.select_container(logical)
|
|
5133
|
+
if selected is not None:
|
|
5134
|
+
return selected
|
|
5135
|
+
# It disappeared between the strict stat and selection; try the twin.
|
|
5136
|
+
return None
|
|
5137
|
+
|
|
5138
|
+
|
|
5139
|
+
def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict | None" = None):
|
|
5065
5140
|
"""Re-read the physical line at ``(row.source_path, row.line_offset)``, validate
|
|
5066
5141
|
it against the stored ``codex_conversation_events.payload_json`` for that exact
|
|
5067
5142
|
position (§3.4 structural gone-check — the canonical FULL record, not
|
|
@@ -5073,19 +5148,49 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row):
|
|
|
5073
5148
|
The full pre-cap content is re-derived through the SAME ``_extract`` the
|
|
5074
5149
|
normalizer uses — which is how payload serves content beyond the normalized
|
|
5075
5150
|
``CODEX_TEXT_CAP``. Truncation/``truncated`` is against ``_FULL_PAYLOAD_CEILING``
|
|
5076
|
-
(1,000,000 Python characters), the same ceiling the Claude payload path uses.
|
|
5151
|
+
(1,000,000 Python characters), the same ceiling the Claude payload path uses.
|
|
5152
|
+
|
|
5153
|
+
#968: select plain first, else zstd, and seek in decoded offsets. Streams are
|
|
5154
|
+
request-local and reusable; no decoded copies persist. Present unreadable
|
|
5155
|
+
containers raise CodexContainerReadError, never a false gone result."""
|
|
5077
5156
|
stored = conn.execute(
|
|
5078
5157
|
"SELECT payload_json FROM codex_conversation_events "
|
|
5079
5158
|
"WHERE source_path = ? AND line_offset = ?",
|
|
5080
5159
|
(row.source_path, row.line_offset)).fetchone()
|
|
5081
5160
|
if stored is None:
|
|
5082
5161
|
return None
|
|
5083
|
-
|
|
5084
|
-
|
|
5085
|
-
|
|
5086
|
-
|
|
5087
|
-
|
|
5088
|
-
|
|
5162
|
+
for attempt in range(2):
|
|
5163
|
+
selected = _select_codex_payload_container(row.source_path)
|
|
5164
|
+
if selected is None:
|
|
5165
|
+
return None
|
|
5166
|
+
# Validate exactly the descriptor we open, including after a compression
|
|
5167
|
+
# race. The caller has already checked the logical path and known root.
|
|
5168
|
+
root_path = _codex_source_root_path(conn, row.source_root_key)
|
|
5169
|
+
if not _within_root(str(selected.path), root_path):
|
|
5170
|
+
raise _CodexPayloadContainmentError()
|
|
5171
|
+
try:
|
|
5172
|
+
fh = None if streams is None else streams.get(str(selected.path))
|
|
5173
|
+
if fh is None:
|
|
5174
|
+
fh = _rc.open_decoded(selected)
|
|
5175
|
+
if streams is not None:
|
|
5176
|
+
streams[str(selected.path)] = fh
|
|
5177
|
+
try:
|
|
5178
|
+
fh.seek(row.line_offset)
|
|
5179
|
+
line = fh.readline()
|
|
5180
|
+
finally:
|
|
5181
|
+
if streams is None:
|
|
5182
|
+
fh.close()
|
|
5183
|
+
break
|
|
5184
|
+
except OSError as exc:
|
|
5185
|
+
# ContainerError subclasses OSError; preserve its approved leaf
|
|
5186
|
+
# category instead of flattening decoder/integrity errors.
|
|
5187
|
+
replacement = _select_codex_payload_container(row.source_path)
|
|
5188
|
+
if replacement is None:
|
|
5189
|
+
return None
|
|
5190
|
+
if replacement.path != selected.path and attempt == 0:
|
|
5191
|
+
continue # plain vanished during Codex's compression window
|
|
5192
|
+
category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
|
|
5193
|
+
raise CodexContainerReadError(category, str(row.source_path)) from exc
|
|
5089
5194
|
try:
|
|
5090
5195
|
obj = json.loads(line.decode("utf-8"))
|
|
5091
5196
|
except (UnicodeDecodeError, ValueError, TypeError):
|
|
@@ -5172,7 +5277,9 @@ def read_codex_payload(
|
|
|
5172
5277
|
full bounded ``card``, and ``truncated`` reflects ``_FULL_PAYLOAD_CEILING``.
|
|
5173
5278
|
``gone`` (→ HTTP 410) means the physical
|
|
5174
5279
|
record moved/mutated; ``not_found`` (→ 404) means no such block, no output
|
|
5175
|
-
partner, or a containment miss (a read is never attempted outside the root).
|
|
5280
|
+
partner, or a containment miss (a read is never attempted outside the root).
|
|
5281
|
+
A present container that cannot be read raises CodexContainerReadError through
|
|
5282
|
+
the existing query-error envelope, including plain-file read failures."""
|
|
5176
5283
|
miss = {"status": "not_found", "block_key": block_key, "which": which}
|
|
5177
5284
|
if which not in ("call", "output", "event"):
|
|
5178
5285
|
return miss
|
|
@@ -5187,7 +5294,14 @@ def read_codex_payload(
|
|
|
5187
5294
|
root_path = _codex_source_root_path(conn, target.source_root_key)
|
|
5188
5295
|
if not _within_root(target.source_path, root_path):
|
|
5189
5296
|
return miss
|
|
5190
|
-
|
|
5297
|
+
streams: dict = {}
|
|
5298
|
+
try:
|
|
5299
|
+
outcome = _reread_codex_full_content(conn, target, streams=streams)
|
|
5300
|
+
except _CodexPayloadContainmentError:
|
|
5301
|
+
return miss
|
|
5302
|
+
finally:
|
|
5303
|
+
for stream in streams.values():
|
|
5304
|
+
stream.close()
|
|
5191
5305
|
if outcome is None:
|
|
5192
5306
|
return {"status": "gone", "block_key": block_key, "which": which}
|
|
5193
5307
|
content, truncated, card = outcome
|
|
@@ -43,15 +43,16 @@ All six §5.4 bullets are contract:
|
|
|
43
43
|
its first ingest attempt (so a discovered file is never lost even if its
|
|
44
44
|
first ingest is dirty/lock-contended and leaves no cursor row). A path is
|
|
45
45
|
retried while unclassified: with no committed cursor it is surfaced every
|
|
46
|
-
time it is visited; with a committed cursor it is surfaced only when its
|
|
47
|
-
|
|
48
|
-
any directory-mtime change). A path leaves the set when the driver ``reap``s
|
|
46
|
+
time it is visited; with a committed cursor it is surfaced only when its
|
|
47
|
+
signature advances beyond that cursor (an incomplete-``session_meta`` tail
|
|
48
|
+
completing without any directory-mtime change). A path leaves the set when the driver ``reap``s
|
|
49
49
|
it (it gained a thread row — child or non-child), when it vanishes, or on a
|
|
50
50
|
bounded per-candidate expiry (handed to the next full sync).
|
|
51
51
|
"""
|
|
52
52
|
from __future__ import annotations
|
|
53
53
|
|
|
54
54
|
import os
|
|
55
|
+
import _lib_codex_rollout_container as _rc
|
|
55
56
|
|
|
56
57
|
|
|
57
58
|
# ── production-default injected callables ─────────────────────────────────────
|
|
@@ -101,6 +102,48 @@ def default_file_size(filepath):
|
|
|
101
102
|
return st.st_size
|
|
102
103
|
|
|
103
104
|
|
|
105
|
+
def default_file_sig(filepath):
|
|
106
|
+
"""The selected container's live-tail signature (#968 spec 5.9).
|
|
107
|
+
|
|
108
|
+
Plain keeps today's size (as ``("plain", size)``); a compressed-only
|
|
109
|
+
rollout reports its container fingerprint instead of vanishing.
|
|
110
|
+
"""
|
|
111
|
+
return _rc.observe_watch_sig(filepath)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _as_sig(value):
|
|
115
|
+
if value is None or isinstance(value, tuple):
|
|
116
|
+
return value
|
|
117
|
+
return (_rc.PLAIN, int(value))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def codex_sig_advanced(observed, committed) -> bool:
|
|
121
|
+
"""Whether an observed signature is ahead of the committed one.
|
|
122
|
+
|
|
123
|
+
Plain grows (append-only, size decides); any other change of a compressed
|
|
124
|
+
container, or of the container kind, is a change. A bare int on either
|
|
125
|
+
side is a legacy plain size.
|
|
126
|
+
"""
|
|
127
|
+
obs, com = _as_sig(observed), _as_sig(committed)
|
|
128
|
+
if obs is None:
|
|
129
|
+
return False
|
|
130
|
+
if com is None:
|
|
131
|
+
return True
|
|
132
|
+
if obs[0] == com[0] == _rc.PLAIN:
|
|
133
|
+
return obs[1] > com[1]
|
|
134
|
+
return obs != com
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def committed_watch_sig(kind, size_bytes, container_size_bytes, mtime_ns,
|
|
138
|
+
device, inode):
|
|
139
|
+
"""The signature a cursor row's ACCEPTED container would observe: the
|
|
140
|
+
leaf's ``ContainerFingerprint.watch_sig`` shape, device included."""
|
|
141
|
+
if kind in (None, _rc.PLAIN):
|
|
142
|
+
return (_rc.PLAIN, int(size_bytes or 0))
|
|
143
|
+
return (_rc.ZSTD, int(container_size_bytes or 0), int(mtime_ns or 0),
|
|
144
|
+
int(device or 0), int(inode or 0))
|
|
145
|
+
|
|
146
|
+
|
|
104
147
|
_SENTINEL = object()
|
|
105
148
|
|
|
106
149
|
|
|
@@ -118,7 +161,7 @@ class CodexChildFrontier:
|
|
|
118
161
|
pending_expiry_cycles: int = 50,
|
|
119
162
|
scandir_fn=default_scandir,
|
|
120
163
|
dir_mtime_fn=default_dir_mtime,
|
|
121
|
-
file_size_fn=
|
|
164
|
+
file_size_fn=default_file_sig,
|
|
122
165
|
clock_fn=None,
|
|
123
166
|
):
|
|
124
167
|
self._walk_root = str(walk_root)
|
|
@@ -175,7 +218,7 @@ class CodexChildFrontier:
|
|
|
175
218
|
|
|
176
219
|
``known_paths`` — every path already tracked in ``codex_session_files``,
|
|
177
220
|
so a newly enumerated file already known to some table is not
|
|
178
|
-
re-registered. ``committed_sizes`` — ``{path:
|
|
221
|
+
re-registered. ``committed_sizes`` — ``{path: signature}`` committed
|
|
179
222
|
cursors, for the pending-growth check.
|
|
180
223
|
|
|
181
224
|
Returns the set of candidate paths to targeted-ingest this cycle (new
|
|
@@ -251,8 +294,8 @@ class CodexChildFrontier:
|
|
|
251
294
|
mtime as the dir's baseline (so a create landing during the enumeration
|
|
252
295
|
is re-examined next rotation), skips symlinked descendants (the
|
|
253
296
|
``follow_symlinks=False`` boundary), adds real subdirectories to the
|
|
254
|
-
frontier, and registers brand-new
|
|
255
|
-
their first ingest — surfacing each for that first ingest attempt in the
|
|
297
|
+
frontier, and registers brand-new logical rollout files in ``pending``
|
|
298
|
+
BEFORE their first ingest — surfacing each for that first ingest attempt in the
|
|
256
299
|
SAME cycle it is discovered (the pending sweep then owns the retries).
|
|
257
300
|
Returns True iff it consumed a filesystem op."""
|
|
258
301
|
if not self._enum_queue:
|
|
@@ -267,10 +310,12 @@ class CodexChildFrontier:
|
|
|
267
310
|
if path not in self._dir_set:
|
|
268
311
|
self._dir_set.add(path)
|
|
269
312
|
self._dirs.append(path) # new subdir joins frontier
|
|
270
|
-
elif
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
313
|
+
elif _rc.is_container_name(name):
|
|
314
|
+
# Both spellings enumerate one logical identity.
|
|
315
|
+
logical = os.fspath(_rc.logical_path(path))
|
|
316
|
+
if logical not in self._known and logical not in self._pending:
|
|
317
|
+
self._pending[logical] = 0 # register BEFORE first ingest
|
|
318
|
+
ingest.add(logical) # first ingest attempt now
|
|
274
319
|
return True
|
|
275
320
|
|
|
276
321
|
def _op_pending(self, ingest: set) -> bool:
|
|
@@ -286,12 +331,12 @@ class CodexChildFrontier:
|
|
|
286
331
|
p = keys[idx]
|
|
287
332
|
if p not in self._pending: # reaped/dropped mid-cycle
|
|
288
333
|
continue
|
|
289
|
-
|
|
290
|
-
if
|
|
334
|
+
sig = self._file_size(p) # 1 signature probe
|
|
335
|
+
if sig is None:
|
|
291
336
|
self._pending.pop(p, None) # vanished → drop
|
|
292
337
|
else:
|
|
293
338
|
committed = self._committed.get(p)
|
|
294
|
-
if
|
|
339
|
+
if codex_sig_advanced(sig, committed):
|
|
295
340
|
ingest.add(p)
|
|
296
341
|
return True
|
|
297
342
|
return False
|