cctally 1.116.1 → 1.117.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +57 -3
- package/bin/_cctally_cache.py +5720 -885
- package/bin/_cctally_claude_attribution.py +997 -0
- package/bin/_cctally_completion.py +101 -28
- package/bin/_cctally_core.py +235 -12
- package/bin/_cctally_dashboard.py +263 -17
- package/bin/_cctally_dashboard_conversation.py +107 -21
- package/bin/_cctally_dashboard_envelope.py +44 -0
- package/bin/_cctally_db.py +928 -80
- package/bin/_cctally_diagnosis_sources.py +78 -13
- package/bin/_cctally_doctor.py +211 -86
- package/bin/_cctally_five_hour.py +446 -22
- package/bin/_cctally_forecast.py +302 -6
- package/bin/_cctally_journal.py +2418 -158
- package/bin/_cctally_milestone_history.py +379 -91
- package/bin/_cctally_oauth_samples.py +185 -7
- package/bin/_cctally_parser.py +38 -0
- package/bin/_cctally_project.py +34 -8
- package/bin/_cctally_quarantine.py +821 -343
- package/bin/_cctally_quota.py +69 -4
- package/bin/_cctally_quota_calibration.py +677 -3
- package/bin/_cctally_quota_model.py +927 -127
- package/bin/_cctally_record.py +922 -445
- package/bin/_cctally_rederive.py +119 -15
- package/bin/_cctally_statusline.py +1048 -176
- package/bin/_cctally_store.py +544 -110
- package/bin/_cctally_tui.py +455 -42
- package/bin/_cctally_update.py +77 -14
- package/bin/_lib_account_protocol.py +227 -19
- package/bin/_lib_claude_attribution.py +421 -0
- package/bin/_lib_clone_copy.py +30 -0
- package/bin/_lib_codex_conversation_query.py +45 -9
- package/bin/_lib_codex_conversation_watch.py +72 -13
- package/bin/_lib_codex_rollout_container.py +234 -19
- package/bin/_lib_codex_worker_lifecycle.py +374 -0
- package/bin/_lib_credit.py +22 -13
- package/bin/_lib_dashboard_sources.py +257 -18
- package/bin/_lib_detector_access_registry.py +1335 -166
- package/bin/_lib_doctor.py +276 -92
- package/bin/_lib_ingest_frontier.py +269 -36
- package/bin/_lib_journal.py +273 -22
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_meter_rate_change.py +72 -0
- package/bin/_lib_outcome_quarantine.py +917 -554
- package/bin/_lib_provider_activity.py +24 -6
- package/bin/_lib_quota_calibration.py +57 -3
- package/bin/_lib_quota_copy.py +7 -2
- package/bin/_lib_quota_lineage.py +613 -0
- package/bin/_lib_quota_model.py +464 -33
- package/bin/_lib_rate_change_delivery.py +16 -0
- package/bin/_lib_reclaim_planner.py +6 -4
- package/bin/_lib_rederive.py +2046 -21
- package/bin/_lib_selector_state.py +131 -3
- package/bin/_lib_snapshot_cache.py +8 -0
- package/bin/_lib_stale_proof.py +113 -4
- package/bin/_lib_statusline.py +72 -3
- package/bin/_lib_statusline_candidates.py +246 -42
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +419 -6
- package/bin/cctally +28 -11
- package/dashboard/static/assets/{ConversationsView-Dflb_myA.js → ConversationsView-oqNsWzWV.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-DFX5CfIs.js → DoctorModal-B0DnU9d9.js} +1 -1
- package/dashboard/static/assets/ModalRoot-CYua2wP_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-C3PJy1xx.js → ProjectsDrillPanel-DSxXQSa-.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-BczsWKhD.js → SourceDetailModal-BleDpQqG.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-Os-Lo5cs.js → UpdateModal-CMqaAkSK.js} +1 -1
- package/dashboard/static/assets/{index-CYZ3NLqF.js → index-BjQV-0-D.js} +3 -3
- package/dashboard/static/assets/index-D34eYhAc.css +1 -0
- package/dashboard/static/assets/{outlineNavigation-k-zIif6W.js → outlineNavigation-CO-VWBm7.js} +1 -1
- package/dashboard/static/assets/{useKeymap-CW7Up0Ay.js → useKeymap-CUgEnnua.js} +1 -1
- package/dashboard/static/dashboard.html +3 -3
- package/package.json +5 -1
- package/dashboard/static/assets/ModalRoot-HlxoagIa.js +0 -1
- package/dashboard/static/assets/index-BKITk1CT.css +0 -1
|
@@ -17,6 +17,9 @@ All six §5.4 bullets are contract:
|
|
|
17
17
|
1. **walk_root-only seed.** The frontier index starts as JUST the configured
|
|
18
18
|
``CodexProviderRoot.walk_root`` — construction walks nothing. Directories are
|
|
19
19
|
discovered incrementally as the frontier is enumerated.
|
|
20
|
+
#970: a native home's archive directory is a second seed, enumerated FLAT
|
|
21
|
+
(its subdirectories never join the frontier) under the same budget, and
|
|
22
|
+
each mappable file registers under its DATED logical path.
|
|
20
23
|
2. **One shared per-cycle operation budget** covering directory ``stat``s,
|
|
21
24
|
directory enumerations, and pending-candidate ``stat``s, with round-robin
|
|
22
25
|
fairness across the three work classes and rotating continuation cursors, so
|
|
@@ -52,6 +55,8 @@ All six §5.4 bullets are contract:
|
|
|
52
55
|
from __future__ import annotations
|
|
53
56
|
|
|
54
57
|
import os
|
|
58
|
+
import pathlib
|
|
59
|
+
|
|
55
60
|
import _lib_codex_rollout_container as _rc
|
|
56
61
|
|
|
57
62
|
|
|
@@ -102,13 +107,20 @@ def default_file_size(filepath):
|
|
|
102
107
|
return st.st_size
|
|
103
108
|
|
|
104
109
|
|
|
105
|
-
def default_file_sig(filepath):
|
|
110
|
+
def default_file_sig(filepath, *, archive_dir=None):
|
|
106
111
|
"""The selected container's live-tail signature (#968 spec 5.9).
|
|
107
112
|
|
|
108
113
|
Plain keeps today's size (as ``("plain", size)``); a compressed-only
|
|
109
|
-
rollout reports its container fingerprint instead of vanishing.
|
|
114
|
+
rollout reports its container fingerprint instead of vanishing. #970: with
|
|
115
|
+
an ``archive_dir`` (a dated rollout of a native home), an archived
|
|
116
|
+
container is observed too, its signature carrying the ``"archive"``
|
|
117
|
+
location; the dated containers are tried first, so a dated rollout costs
|
|
118
|
+
exactly today's probe.
|
|
110
119
|
"""
|
|
111
|
-
|
|
120
|
+
sig = _rc.observe_watch_sig(filepath)
|
|
121
|
+
if sig is not None or archive_dir is None:
|
|
122
|
+
return sig
|
|
123
|
+
return _rc.observe_watch_sig(filepath, archive_dir=archive_dir)
|
|
112
124
|
|
|
113
125
|
|
|
114
126
|
def _as_sig(value):
|
|
@@ -122,26 +134,33 @@ def codex_sig_advanced(observed, committed) -> bool:
|
|
|
122
134
|
|
|
123
135
|
Plain grows (append-only, size decides); any other change of a compressed
|
|
124
136
|
container, or of the container kind, is a change. A bare int on either
|
|
125
|
-
side is a legacy plain size.
|
|
137
|
+
side is a legacy plain size. #970: a plain container at a different
|
|
138
|
+
location (the trailing ``"archive"`` element) is a change whatever its
|
|
139
|
+
size, so a pure archive move is observed once.
|
|
126
140
|
"""
|
|
127
141
|
obs, com = _as_sig(observed), _as_sig(committed)
|
|
128
142
|
if obs is None:
|
|
129
143
|
return False
|
|
130
144
|
if com is None:
|
|
131
145
|
return True
|
|
132
|
-
if obs[0] == com[0] == _rc.PLAIN:
|
|
146
|
+
if obs[0] == com[0] == _rc.PLAIN and obs[2:] == com[2:]:
|
|
133
147
|
return obs[1] > com[1]
|
|
134
148
|
return obs != com
|
|
135
149
|
|
|
136
150
|
|
|
137
151
|
def committed_watch_sig(kind, size_bytes, container_size_bytes, mtime_ns,
|
|
138
|
-
device, inode):
|
|
152
|
+
device, inode, location=_rc.DATED):
|
|
139
153
|
"""The signature a cursor row's ACCEPTED container would observe: the
|
|
140
|
-
leaf's ``ContainerFingerprint.watch_sig`` shape, device included
|
|
154
|
+
leaf's ``ContainerFingerprint.watch_sig`` shape, device included, with
|
|
155
|
+
the location appended exactly when the leaf appends it (#970: only for a
|
|
156
|
+
container outside the dated location, so every dated signature keeps
|
|
157
|
+
today's shape)."""
|
|
141
158
|
if kind in (None, _rc.PLAIN):
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
159
|
+
sig = (_rc.PLAIN, int(size_bytes or 0))
|
|
160
|
+
else:
|
|
161
|
+
sig = (_rc.ZSTD, int(container_size_bytes or 0), int(mtime_ns or 0),
|
|
162
|
+
int(device or 0), int(inode or 0))
|
|
163
|
+
return sig if location == _rc.DATED else sig + (str(location),)
|
|
145
164
|
|
|
146
165
|
|
|
147
166
|
_SENTINEL = object()
|
|
@@ -157,24 +176,37 @@ class CodexChildFrontier:
|
|
|
157
176
|
self,
|
|
158
177
|
walk_root,
|
|
159
178
|
*,
|
|
179
|
+
archive_root=None,
|
|
160
180
|
op_budget: int = 64,
|
|
161
181
|
pending_expiry_cycles: int = 50,
|
|
162
182
|
scandir_fn=default_scandir,
|
|
163
183
|
dir_mtime_fn=default_dir_mtime,
|
|
164
|
-
file_size_fn=
|
|
184
|
+
file_size_fn=None,
|
|
165
185
|
clock_fn=None,
|
|
166
186
|
):
|
|
167
187
|
self._walk_root = str(walk_root)
|
|
188
|
+
# #970 spec 5.8: a native home's archive directory is a second seeded
|
|
189
|
+
# directory, enumerated FLAT under the same operation budget; each
|
|
190
|
+
# mappable file registers under its dated logical path.
|
|
191
|
+
self._archive_root = None if archive_root is None else str(archive_root)
|
|
192
|
+
self._locations = (_rc.CodexLocations(
|
|
193
|
+
pathlib.Path(self._walk_root),
|
|
194
|
+
None if self._archive_root is None
|
|
195
|
+
else pathlib.Path(self._archive_root)),)
|
|
168
196
|
self._op_budget = max(1, int(op_budget))
|
|
169
197
|
self._pending_expiry = max(1, int(pending_expiry_cycles))
|
|
170
198
|
self._scandir = scandir_fn
|
|
171
199
|
self._dir_mtime = dir_mtime_fn
|
|
172
|
-
self._file_size = file_size_fn
|
|
200
|
+
self._file_size = file_size_fn or self._default_file_sig
|
|
173
201
|
self._clock = clock_fn or (lambda: 0)
|
|
174
202
|
|
|
175
|
-
# Frontier index — seeded with JUST walk_root (
|
|
203
|
+
# Frontier index — seeded with JUST walk_root (and the archive
|
|
204
|
+
# directory, when there is one); no enumeration at start.
|
|
176
205
|
self._dirs: list[str] = [self._walk_root]
|
|
177
206
|
self._dir_set: set[str] = {self._walk_root}
|
|
207
|
+
if self._archive_root is not None:
|
|
208
|
+
self._dirs.append(self._archive_root)
|
|
209
|
+
self._dir_set.add(self._archive_root)
|
|
178
210
|
# dir -> mtime observed when it was last enumerated (absent => never).
|
|
179
211
|
self._enum_mtime: dict[str, int] = {}
|
|
180
212
|
# dirs awaiting a scandir, each carried with the mtime that triggered it.
|
|
@@ -190,6 +222,12 @@ class CodexChildFrontier:
|
|
|
190
222
|
self._completed_rotation_watermark = None
|
|
191
223
|
self._rotation_count = 0
|
|
192
224
|
|
|
225
|
+
def _default_file_sig(self, path):
|
|
226
|
+
"""The production signature probe, archive candidates included."""
|
|
227
|
+
archive_dir = (None if self._archive_root is None
|
|
228
|
+
else _rc.archive_dir_for(path, self._locations))
|
|
229
|
+
return default_file_sig(path, archive_dir=archive_dir)
|
|
230
|
+
|
|
193
231
|
# ── introspection (tests / diagnostics) ──────────────────────────────────
|
|
194
232
|
|
|
195
233
|
def known_directories(self) -> set[str]:
|
|
@@ -303,6 +341,9 @@ class CodexChildFrontier:
|
|
|
303
341
|
d, mt = self._enum_queue.pop(0)
|
|
304
342
|
entries = self._scandir(d) # 1 fs op
|
|
305
343
|
self._enum_mtime[d] = mt
|
|
344
|
+
if d == self._archive_root:
|
|
345
|
+
self._register_archive_entries(entries, ingest)
|
|
346
|
+
return True
|
|
306
347
|
for (name, path, is_dir, is_sym) in entries:
|
|
307
348
|
if is_sym:
|
|
308
349
|
continue # never traverse a symlink
|
|
@@ -318,6 +359,24 @@ class CodexChildFrontier:
|
|
|
318
359
|
ingest.add(logical) # first ingest attempt now
|
|
319
360
|
return True
|
|
320
361
|
|
|
362
|
+
def _register_archive_entries(self, entries, ingest: set) -> None:
|
|
363
|
+
"""Register the archive's mappable files under their dated logical
|
|
364
|
+
paths (#970 spec 5.3, 5.8). Flat: a subdirectory never joins the
|
|
365
|
+
frontier. An unmappable name has no logical identity and is skipped.
|
|
366
|
+
A symlinked entry is skipped exactly as under the walk root (bullet
|
|
367
|
+
4): this discovery is a fast path, and the full sync still reaches a
|
|
368
|
+
linked rollout under its canonical identity."""
|
|
369
|
+
for (name, path, is_dir, is_sym) in entries:
|
|
370
|
+
if is_sym or is_dir or not _rc.is_container_name(name):
|
|
371
|
+
continue
|
|
372
|
+
dated = _rc.dated_logical_for_archive(path, self._locations)
|
|
373
|
+
if dated is None:
|
|
374
|
+
continue
|
|
375
|
+
logical = os.fspath(dated)
|
|
376
|
+
if logical not in self._known and logical not in self._pending:
|
|
377
|
+
self._pending[logical] = 0 # register BEFORE first ingest
|
|
378
|
+
ingest.add(logical) # first ingest attempt now
|
|
379
|
+
|
|
321
380
|
def _op_pending(self, ingest: set) -> bool:
|
|
322
381
|
"""Check one pending candidate (rotating cursor). Vanished → dropped; no
|
|
323
382
|
committed cursor (brand-new / lock-contended first ingest) → surfaced for
|
|
@@ -5,7 +5,12 @@ cache, journal, configuration or CLI modules. It owns
|
|
|
5
5
|
|
|
6
6
|
* logical normalization (``X.jsonl.zst`` -> ``X.jsonl``) — every persisted key is
|
|
7
7
|
the logical path, whatever container holds the bytes;
|
|
8
|
-
* container selection (plain wins, else the ``.zst`` twin, else
|
|
8
|
+
* container selection (plain wins, else the ``.zst`` twin, else — when the
|
|
9
|
+
caller supplies an archive directory (#970) — the archived plain, then the
|
|
10
|
+
archived ``.zst``, else none), plus a strict tri-state presence probe;
|
|
11
|
+
* native-home recognition and the pure archive-name mapping (#970 spec 5.1,
|
|
12
|
+
5.2). The leaf never guesses a layout: an archive location is used only when
|
|
13
|
+
a caller passes it explicitly;
|
|
9
14
|
* the call-time decoder chain (stdlib ``compression.zstd`` on 3.14+, then
|
|
10
15
|
``backports.zstd``, then a typed unavailable result);
|
|
11
16
|
* a decoded binary stream whose ``tell``/``seek`` speak DECODED offsets, so a
|
|
@@ -19,13 +24,20 @@ from __future__ import annotations
|
|
|
19
24
|
import hashlib
|
|
20
25
|
import io
|
|
21
26
|
import json
|
|
27
|
+
import datetime as _dt
|
|
28
|
+
import errno as _errno
|
|
22
29
|
import os
|
|
23
30
|
import pathlib
|
|
31
|
+
import re
|
|
24
32
|
import stat as _stat
|
|
25
|
-
from typing import Callable, NamedTuple, Optional, Union
|
|
33
|
+
from typing import Callable, Iterable, NamedTuple, Optional, Union
|
|
26
34
|
|
|
27
35
|
PLAIN = "plain"
|
|
28
36
|
ZSTD = "zstd"
|
|
37
|
+
DATED = "dated"
|
|
38
|
+
ARCHIVE = "archive"
|
|
39
|
+
SESSIONS_DIR = "sessions"
|
|
40
|
+
ARCHIVE_DIR = "archived_sessions"
|
|
29
41
|
PLAIN_SUFFIX = ".jsonl"
|
|
30
42
|
ZSTD_SUFFIX = ".jsonl.zst"
|
|
31
43
|
DECODE_CHUNK = 1 << 20
|
|
@@ -126,15 +138,23 @@ class ContainerFingerprint(NamedTuple):
|
|
|
126
138
|
device: int
|
|
127
139
|
inode: int
|
|
128
140
|
|
|
129
|
-
def watch_sig(self) -> tuple:
|
|
141
|
+
def watch_sig(self, location: str = DATED) -> tuple:
|
|
130
142
|
"""The live-tail signature: plain keeps today's size; zstd binds the
|
|
131
143
|
container's identity, device included, exactly as the zstd generation
|
|
132
144
|
token and every compressed container-unchanged check do (review P3-2,
|
|
133
|
-
round-2 N9), so a remounted container is observed as a change once.
|
|
145
|
+
round-2 N9), so a remounted container is observed as a change once.
|
|
146
|
+
|
|
147
|
+
#970 A8: a container outside the dated location carries its location
|
|
148
|
+
as a trailing element, so a pure archive move (same size and stat
|
|
149
|
+
identity) is observed once. A dated container keeps today's shape
|
|
150
|
+
exactly, so every committed signature stays valid.
|
|
151
|
+
"""
|
|
134
152
|
if self.kind == PLAIN:
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
153
|
+
sig = (PLAIN, int(self.size))
|
|
154
|
+
else:
|
|
155
|
+
sig = (ZSTD, int(self.size), int(self.mtime_ns), int(self.device),
|
|
156
|
+
int(self.inode))
|
|
157
|
+
return sig if location == DATED else sig + (str(location),)
|
|
138
158
|
|
|
139
159
|
def to_json(self) -> str:
|
|
140
160
|
return json.dumps(list(self), separators=(",", ":"))
|
|
@@ -173,24 +193,219 @@ class SelectedContainer(NamedTuple):
|
|
|
173
193
|
kind: str
|
|
174
194
|
path: pathlib.Path
|
|
175
195
|
fingerprint: ContainerFingerprint
|
|
196
|
+
#: #970: ``"dated"`` (beside the logical path) or ``"archive"`` (the
|
|
197
|
+
#: caller-supplied archive directory). Trailing with a default, so every
|
|
198
|
+
#: four-field construction stays valid.
|
|
199
|
+
location: str = DATED
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class CodexLocations(NamedTuple):
|
|
203
|
+
"""One configured Codex root's container locations (#970 spec 5.1).
|
|
204
|
+
|
|
205
|
+
``walk_root`` is the directory discovery walks and every logical path lives
|
|
206
|
+
under; ``archive_root`` is a native home's flat archive directory, or None
|
|
207
|
+
for a direct directory. Either may not exist yet.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
walk_root: pathlib.Path
|
|
211
|
+
archive_root: "pathlib.Path | None"
|
|
212
|
+
|
|
176
213
|
|
|
214
|
+
def home_locations(configured: PathLike) -> "CodexLocations | None":
|
|
215
|
+
"""Native-home recognition (#970 spec 5.1, A7).
|
|
216
|
+
|
|
217
|
+
A configured directory with a ``sessions/`` or an ``archived_sessions/``
|
|
218
|
+
child is a native home: ``walk_root = <home>/sessions`` and
|
|
219
|
+
``archive_root = <home>/archived_sessions``, either possibly missing, so
|
|
220
|
+
the first unarchive (which creates ``sessions/``) never changes an
|
|
221
|
+
identity. Any other existing directory is walked directly with no archive.
|
|
222
|
+
Anything else is not a root.
|
|
223
|
+
"""
|
|
224
|
+
home = pathlib.Path(configured)
|
|
225
|
+
sessions, archive = home / SESSIONS_DIR, home / ARCHIVE_DIR
|
|
226
|
+
try:
|
|
227
|
+
native = sessions.is_dir() or archive.is_dir()
|
|
228
|
+
except OSError:
|
|
229
|
+
native = False
|
|
230
|
+
if native:
|
|
231
|
+
return CodexLocations(sessions, archive)
|
|
232
|
+
try:
|
|
233
|
+
if home.is_dir():
|
|
234
|
+
return CodexLocations(home, None)
|
|
235
|
+
except OSError:
|
|
236
|
+
pass
|
|
237
|
+
return None
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def release_walked_archives(
|
|
241
|
+
locations: "Iterable[CodexLocations]") -> "list[CodexLocations]":
|
|
242
|
+
"""Drop each archive another configured root already walks (#970 N4).
|
|
243
|
+
|
|
244
|
+
``CODEX_HOME=<home>,<home>/archived_sessions`` names one directory twice:
|
|
245
|
+
as the home's archive and as a direct root. The direct root spells its
|
|
246
|
+
identities ``<archive>/X.jsonl``, which the home would read as a retained
|
|
247
|
+
direct-era spelling and hold its dated twins forever. A native home whose
|
|
248
|
+
canonical archive equals, or lies inside, another location's canonical
|
|
249
|
+
walk root therefore gets ``archive_root=None``: the root that walks the
|
|
250
|
+
directory keeps it, exactly as without #970. Order and every other field
|
|
251
|
+
are kept; canonicalization follows symlinks without requiring existence.
|
|
252
|
+
"""
|
|
253
|
+
locations = list(locations)
|
|
254
|
+
|
|
255
|
+
def canonical(path):
|
|
256
|
+
try:
|
|
257
|
+
return pathlib.Path(os.path.realpath(path))
|
|
258
|
+
except (OSError, ValueError):
|
|
259
|
+
return pathlib.Path(os.path.abspath(path))
|
|
260
|
+
|
|
261
|
+
walks = [canonical(location.walk_root) for location in locations]
|
|
262
|
+
out: "list[CodexLocations]" = []
|
|
263
|
+
for index, location in enumerate(locations):
|
|
264
|
+
if location.archive_root is not None:
|
|
265
|
+
archive = canonical(location.archive_root)
|
|
266
|
+
if any(other != index and (archive == walk or walk in archive.parents)
|
|
267
|
+
for other, walk in enumerate(walks)):
|
|
268
|
+
location = location._replace(archive_root=None)
|
|
269
|
+
out.append(location)
|
|
270
|
+
return out
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# The UUID shape `_lib_jsonl._CODEX_FILENAME_UUID_RE` uses; duplicated here
|
|
274
|
+
# because the leaf imports only the standard library.
|
|
275
|
+
_ARCHIVE_NAME_RE = re.compile(
|
|
276
|
+
r"rollout-(\d{4})-(\d{2})-(\d{2})T\d{2}-\d{2}-\d{2}-[0-9a-fA-F-]{36}"
|
|
277
|
+
r"\.jsonl(?:\.zst)?")
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def archive_name_date(name: str) -> "tuple[str, str, str] | None":
|
|
281
|
+
"""``(YYYY, MM, DD)`` of a complete rollout basename, else None (#970 5.2).
|
|
282
|
+
|
|
283
|
+
Only ``rollout-YYYY-MM-DDThh-mm-ss-<uuid>.jsonl[.zst]`` with a valid
|
|
284
|
+
calendar date qualifies. The date is the filename's literal date: no
|
|
285
|
+
timezone conversion.
|
|
286
|
+
"""
|
|
287
|
+
match = _ARCHIVE_NAME_RE.fullmatch(str(name))
|
|
288
|
+
if match is None:
|
|
289
|
+
return None
|
|
290
|
+
year, month, day = match.group(1), match.group(2), match.group(3)
|
|
291
|
+
try:
|
|
292
|
+
_dt.date(int(year), int(month), int(day))
|
|
293
|
+
except ValueError:
|
|
294
|
+
return None
|
|
295
|
+
return year, month, day
|
|
177
296
|
|
|
178
|
-
|
|
179
|
-
|
|
297
|
+
|
|
298
|
+
def archive_dir_for(logical: PathLike,
|
|
299
|
+
locations: "Iterable[CodexLocations]") -> "pathlib.Path | None":
|
|
300
|
+
"""The archive directory whose containers belong to ``logical`` (#970 G2).
|
|
301
|
+
|
|
302
|
+
Pure path arithmetic, no I/O: only a logical path that is exactly
|
|
303
|
+
``walk_root/YYYY/MM/DD/<name>.jsonl`` of a location with an archive, and
|
|
304
|
+
whose name's date is that ``YYYY/MM/DD``, has archive candidates.
|
|
305
|
+
"""
|
|
306
|
+
lp = pathlib.Path(logical_path(os.fspath(logical)))
|
|
307
|
+
if not lp.name.endswith(PLAIN_SUFFIX):
|
|
308
|
+
return None
|
|
309
|
+
date = archive_name_date(lp.name)
|
|
310
|
+
if date is None:
|
|
311
|
+
return None
|
|
312
|
+
day = lp.parent
|
|
313
|
+
if (day.parent.parent.name, day.parent.name, day.name) != date:
|
|
314
|
+
return None
|
|
315
|
+
base = day.parent.parent.parent
|
|
316
|
+
for location in locations:
|
|
317
|
+
if location.archive_root is not None and base == location.walk_root:
|
|
318
|
+
return location.archive_root
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def dated_logical_for_archive(
|
|
323
|
+
container: PathLike,
|
|
324
|
+
locations: "Iterable[CodexLocations]") -> "pathlib.Path | None":
|
|
325
|
+
"""Map ``archive_root/<name>[.zst]`` to ``walk_root/YYYY/MM/DD/<name>``.
|
|
326
|
+
|
|
327
|
+
None for an unmappable name or a path that is not a direct child of any
|
|
328
|
+
location's archive directory (archive discovery is flat).
|
|
329
|
+
"""
|
|
330
|
+
cp = pathlib.Path(container)
|
|
331
|
+
date = archive_name_date(cp.name)
|
|
332
|
+
if date is None:
|
|
333
|
+
return None
|
|
334
|
+
for location in locations:
|
|
335
|
+
if location.archive_root is not None and cp.parent == location.archive_root:
|
|
336
|
+
return location.walk_root.joinpath(*date, logical_path(cp.name))
|
|
337
|
+
return None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def container_candidates(
|
|
341
|
+
logical: PathLike, *, archive_dir: "PathLike | None" = None,
|
|
342
|
+
) -> "list[tuple[str, str, pathlib.Path]]":
|
|
343
|
+
"""``(kind, location, path)`` in selection order (#970 G2)."""
|
|
344
|
+
lp = pathlib.Path(logical_path(os.fspath(logical)))
|
|
345
|
+
out = [(PLAIN, DATED, lp), (ZSTD, DATED, zstd_twin(lp))]
|
|
346
|
+
if archive_dir is not None:
|
|
347
|
+
archive = pathlib.Path(archive_dir)
|
|
348
|
+
out.append((PLAIN, ARCHIVE, archive / lp.name))
|
|
349
|
+
out.append((ZSTD, ARCHIVE, archive / (lp.name + ".zst")))
|
|
350
|
+
return out
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def select_container(logical: PathLike, *,
|
|
354
|
+
archive_dir: "PathLike | None" = None,
|
|
355
|
+
) -> "SelectedContainer | None":
|
|
356
|
+
"""The first regular container in selection order, else None.
|
|
357
|
+
|
|
358
|
+
Plain when it exists, otherwise the ``.zst`` twin (#968 5.2); with an
|
|
359
|
+
explicit ``archive_dir``, then ``archive_dir/<name>.jsonl`` and
|
|
360
|
+
``archive_dir/<name>.jsonl.zst`` (#970 5.2). Lenient: a stat error reads as
|
|
361
|
+
absent. Destructive callers use ``probe_containers`` instead.
|
|
362
|
+
"""
|
|
180
363
|
lp = pathlib.Path(logical_path(os.fspath(logical)))
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
fp = fingerprint_of(twin, ZSTD)
|
|
186
|
-
if fp is not None:
|
|
187
|
-
return SelectedContainer(lp, ZSTD, twin, fp)
|
|
364
|
+
for kind, location, path in container_candidates(lp, archive_dir=archive_dir):
|
|
365
|
+
fp = fingerprint_of(path, kind)
|
|
366
|
+
if fp is not None:
|
|
367
|
+
return SelectedContainer(lp, kind, path, fp, location)
|
|
188
368
|
return None
|
|
189
369
|
|
|
190
370
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
371
|
+
_ABSENT_ERRNOS = frozenset({_errno.ENOENT, _errno.ENOTDIR})
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def probe_containers(logical: PathLike, *,
|
|
375
|
+
archive_dir: "PathLike | None" = None,
|
|
376
|
+
) -> "tuple[str, SelectedContainer | None]":
|
|
377
|
+
"""Strict tri-state presence (#970 spec 5.2, A4).
|
|
378
|
+
|
|
379
|
+
``("present", selected)`` for the first regular candidate in selection
|
|
380
|
+
order; ``("absent", None)`` only when EVERY candidate's stat failed with
|
|
381
|
+
``ENOENT``/``ENOTDIR``; ``("unknown", None)`` otherwise — any other stat
|
|
382
|
+
error (an unreadable directory) or a candidate that exists but is not a
|
|
383
|
+
regular file. Absence is never inferred from an error.
|
|
384
|
+
"""
|
|
385
|
+
lp = pathlib.Path(logical_path(os.fspath(logical)))
|
|
386
|
+
unknown = False
|
|
387
|
+
for kind, location, path in container_candidates(lp, archive_dir=archive_dir):
|
|
388
|
+
try:
|
|
389
|
+
st = os.stat(path)
|
|
390
|
+
except OSError as exc:
|
|
391
|
+
if exc.errno not in _ABSENT_ERRNOS:
|
|
392
|
+
unknown = True
|
|
393
|
+
continue
|
|
394
|
+
if not _stat.S_ISREG(st.st_mode):
|
|
395
|
+
unknown = True
|
|
396
|
+
continue
|
|
397
|
+
fp = ContainerFingerprint(kind, os.fspath(path), int(st.st_size),
|
|
398
|
+
int(st.st_mtime_ns), int(st.st_dev),
|
|
399
|
+
int(st.st_ino))
|
|
400
|
+
return "present", SelectedContainer(lp, kind, path, fp, location)
|
|
401
|
+
return ("unknown", None) if unknown else ("absent", None)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def observe_watch_sig(logical: PathLike, *,
|
|
405
|
+
archive_dir: "PathLike | None" = None) -> "tuple | None":
|
|
406
|
+
selected = select_container(logical, archive_dir=archive_dir)
|
|
407
|
+
return (None if selected is None
|
|
408
|
+
else selected.fingerprint.watch_sig(selected.location))
|
|
194
409
|
|
|
195
410
|
|
|
196
411
|
class DecodedZstdStream:
|