cctally 1.114.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +4394 -279
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +178 -2
- package/bin/_cctally_dashboard.py +2 -2
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_sources.py +18 -9
- package/bin/_cctally_db.py +316 -5
- package/bin/_cctally_doctor.py +436 -27
- package/bin/_cctally_five_hour.py +82 -31
- package/bin/_cctally_forecast.py +81 -9
- package/bin/_cctally_journal.py +3884 -112
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +26 -4
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +677 -149
- package/bin/_cctally_rederive.py +297 -32
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +23 -2
- package/bin/_cctally_statusline.py +6137 -517
- package/bin/_cctally_store.py +287 -48
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +57 -53
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation_query.py +133 -19
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +11 -3
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +452 -1
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +29 -302
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_pricing.py +7 -2
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_rederive.py +94 -5
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/cctally +23 -1
- package/package.json +14 -1
|
@@ -27,11 +27,13 @@ import json
|
|
|
27
27
|
import os
|
|
28
28
|
import re
|
|
29
29
|
import sqlite3
|
|
30
|
+
import stat
|
|
30
31
|
import threading
|
|
31
32
|
from collections import deque
|
|
32
33
|
from types import MappingProxyType
|
|
33
34
|
|
|
34
35
|
import _lib_codex_conversation as kern
|
|
36
|
+
import _lib_codex_rollout_container as _rc
|
|
35
37
|
import _lib_codex_landmarks as landmarks
|
|
36
38
|
import _lib_codex_segments as segkern
|
|
37
39
|
from _lib_codex_find_projection import (
|
|
@@ -1863,6 +1865,29 @@ def codex_conversation_exists(conn: sqlite3.Connection, conversation_key: str) -
|
|
|
1863
1865
|
return row is not None
|
|
1864
1866
|
|
|
1865
1867
|
|
|
1868
|
+
def codex_publication_pending(
|
|
1869
|
+
conn: sqlite3.Connection, conversation_key: str,
|
|
1870
|
+
) -> bool:
|
|
1871
|
+
"""A barrier-population conversation with no publication yet (#968 5.11).
|
|
1872
|
+
|
|
1873
|
+
Such a conversation stays in the existing pending-state handling until it
|
|
1874
|
+
has an eligible publication: it answers ``normalization_pending``, never
|
|
1875
|
+
``not_found``. Any store error answers False (the caller's ``not_found``).
|
|
1876
|
+
"""
|
|
1877
|
+
try:
|
|
1878
|
+
if conn.execute(
|
|
1879
|
+
"SELECT 1 FROM cache_db.codex_barrier_population "
|
|
1880
|
+
"WHERE conversation_key=?", (conversation_key,)).fetchone() is None:
|
|
1881
|
+
return False
|
|
1882
|
+
row = conn.execute(
|
|
1883
|
+
"SELECT published_revision, legacy_uncertified "
|
|
1884
|
+
"FROM main.codex_conversation_publications WHERE conversation_key=?",
|
|
1885
|
+
(conversation_key,)).fetchone()
|
|
1886
|
+
except sqlite3.Error:
|
|
1887
|
+
return False
|
|
1888
|
+
return row is None or (int(row[0] or 0) == 0 and not row[1])
|
|
1889
|
+
|
|
1890
|
+
|
|
1866
1891
|
def codex_conversation_source_paths(
|
|
1867
1892
|
conn: sqlite3.Connection, conversation_key: str
|
|
1868
1893
|
) -> list[str]:
|
|
@@ -2345,6 +2370,9 @@ def get_codex_conversation(
|
|
|
2345
2370
|
# ── Phase A: the narrow index pass ───────────────────────────────────────
|
|
2346
2371
|
rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
|
|
2347
2372
|
if not rows:
|
|
2373
|
+
if codex_publication_pending(conn, conversation_key):
|
|
2374
|
+
return {"status": "normalization_pending",
|
|
2375
|
+
"conversation_key": conversation_key, "items": [], "children": []}
|
|
2348
2376
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
2349
2377
|
undecodable_keys = undecodable_codex_conversation_keys(conn)
|
|
2350
2378
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
@@ -3035,6 +3063,9 @@ def _outline_envelope(
|
|
|
3035
3063
|
# The outline is not the route F3 bounds, and it stays turn-granular.
|
|
3036
3064
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
3037
3065
|
if not rows:
|
|
3066
|
+
if codex_publication_pending(conn, conversation_key):
|
|
3067
|
+
return {"status": "normalization_pending",
|
|
3068
|
+
"conversation_key": conversation_key, "turns": [], "files": [], "children": []}
|
|
3038
3069
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
3039
3070
|
detail_bytes = _detail_bytes_of(rows)
|
|
3040
3071
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
@@ -3421,20 +3452,26 @@ def _stored_browse_page_queries(
|
|
|
3421
3452
|
"(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
|
|
3422
3453
|
page_params.extend((cursor_last, cursor_last, cursor_key))
|
|
3423
3454
|
|
|
3455
|
+
# Keep native/parent enrichment inside the page statement. A LEFT JOIN
|
|
3456
|
+
# against the compound last-good view materializes retained history;
|
|
3457
|
+
# correlated probes push these predicates into its indexed branches.
|
|
3458
|
+
# The scalar parent lookup follows the detail reader's same-root lookup.
|
|
3459
|
+
parent_key_sql = (
|
|
3460
|
+
"(SELECT pt.conversation_key FROM codex_conversation_threads pt "
|
|
3461
|
+
"WHERE pt.source_root_key = r.source_root_key "
|
|
3462
|
+
"AND pt.native_thread_id = r.parent_thread_id "
|
|
3463
|
+
"AND pt.conversation_key != r.conversation_key LIMIT 1)"
|
|
3464
|
+
)
|
|
3424
3465
|
sql = (
|
|
3425
3466
|
"SELECT r.conversation_key, r.item_count, r.started_utc, "
|
|
3426
3467
|
"r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
|
|
3427
|
-
"r.title, r.parent_thread_id, r.source_root_key,
|
|
3428
|
-
"
|
|
3468
|
+
"r.title, r.parent_thread_id, r.source_root_key, "
|
|
3469
|
+
"(SELECT t.native_thread_id FROM codex_conversation_threads t "
|
|
3470
|
+
"WHERE t.conversation_key = r.conversation_key), "
|
|
3471
|
+
f"{parent_key_sql}, pr.title, pr.project_label, r.parent_thread_id "
|
|
3429
3472
|
"FROM codex_conversation_rollups r "
|
|
3430
|
-
"LEFT JOIN codex_conversation_threads t "
|
|
3431
|
-
"ON t.conversation_key = r.conversation_key "
|
|
3432
|
-
"LEFT JOIN codex_conversation_threads pt "
|
|
3433
|
-
"ON pt.source_root_key = r.source_root_key "
|
|
3434
|
-
"AND pt.native_thread_id = r.parent_thread_id "
|
|
3435
|
-
"AND pt.conversation_key != r.conversation_key "
|
|
3436
3473
|
"LEFT JOIN codex_conversation_rollups pr "
|
|
3437
|
-
"ON pr.conversation_key =
|
|
3474
|
+
f"ON pr.conversation_key = {parent_key_sql} "
|
|
3438
3475
|
f"WHERE {' AND '.join(page_where)} "
|
|
3439
3476
|
"ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
|
|
3440
3477
|
)
|
|
@@ -4962,6 +4999,8 @@ def find_in_codex_conversation(
|
|
|
4962
4999
|
return {**base, "status": "normalization_pending"}
|
|
4963
5000
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
4964
5001
|
if not rows:
|
|
5002
|
+
if codex_publication_pending(conn, conversation_key):
|
|
5003
|
+
return {**base, "status": "normalization_pending"}
|
|
4965
5004
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
4966
5005
|
q = (query or "").strip()
|
|
4967
5006
|
if not q:
|
|
@@ -5021,6 +5060,9 @@ def codex_conversation_prompts(conn: sqlite3.Connection, conversation_key: str)
|
|
|
5021
5060
|
"conversation_key": conversation_key, "prompts": []}
|
|
5022
5061
|
rows = _load_conversation_rows(conn, conversation_key)
|
|
5023
5062
|
if not rows:
|
|
5063
|
+
if codex_publication_pending(conn, conversation_key):
|
|
5064
|
+
return {"status": "normalization_pending",
|
|
5065
|
+
"conversation_key": conversation_key, "prompts": []}
|
|
5024
5066
|
return {"status": "not_found", "conversation_key": conversation_key}
|
|
5025
5067
|
kept, _suppressed = kern.pair_mirrors(rows)
|
|
5026
5068
|
items = kern.canonical_items(kept)
|
|
@@ -5061,7 +5103,40 @@ def _within_root(source_path: str | None, root_path: str | None) -> bool:
|
|
|
5061
5103
|
return False
|
|
5062
5104
|
|
|
5063
5105
|
|
|
5064
|
-
|
|
5106
|
+
class CodexContainerReadError(RuntimeError):
|
|
5107
|
+
"""A present rollout failed to read; the query envelope reports HTTP 500."""
|
|
5108
|
+
|
|
5109
|
+
def __init__(self, category: str, logical: str) -> None:
|
|
5110
|
+
super().__init__(f"Codex rollout could not be read ({category})")
|
|
5111
|
+
self.category = category
|
|
5112
|
+
self.logical = logical
|
|
5113
|
+
|
|
5114
|
+
|
|
5115
|
+
class _CodexPayloadContainmentError(RuntimeError):
|
|
5116
|
+
"""Internal 404 signal: the selected physical container escaped its root."""
|
|
5117
|
+
|
|
5118
|
+
|
|
5119
|
+
def _select_codex_payload_container(logical):
|
|
5120
|
+
# The discovery leaf deliberately treats stat failures as absent. A payload
|
|
5121
|
+
# request must distinguish a missing container (410) from inaccessible data
|
|
5122
|
+
# (500), including an inaccessible plain file beside a readable zstd twin.
|
|
5123
|
+
for candidate in (logical, _rc.zstd_twin(logical)):
|
|
5124
|
+
try:
|
|
5125
|
+
info = os.stat(candidate)
|
|
5126
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
5127
|
+
continue
|
|
5128
|
+
except OSError as exc:
|
|
5129
|
+
raise CodexContainerReadError("read-error", str(logical)) from exc
|
|
5130
|
+
if not stat.S_ISREG(info.st_mode):
|
|
5131
|
+
raise CodexContainerReadError("read-error", str(logical))
|
|
5132
|
+
selected = _rc.select_container(logical)
|
|
5133
|
+
if selected is not None:
|
|
5134
|
+
return selected
|
|
5135
|
+
# It disappeared between the strict stat and selection; try the twin.
|
|
5136
|
+
return None
|
|
5137
|
+
|
|
5138
|
+
|
|
5139
|
+
def _reread_codex_full_content(conn: sqlite3.Connection, row, *, streams: "dict | None" = None):
|
|
5065
5140
|
"""Re-read the physical line at ``(row.source_path, row.line_offset)``, validate
|
|
5066
5141
|
it against the stored ``codex_conversation_events.payload_json`` for that exact
|
|
5067
5142
|
position (§3.4 structural gone-check — the canonical FULL record, not
|
|
@@ -5073,19 +5148,49 @@ def _reread_codex_full_content(conn: sqlite3.Connection, row):
|
|
|
5073
5148
|
The full pre-cap content is re-derived through the SAME ``_extract`` the
|
|
5074
5149
|
normalizer uses — which is how payload serves content beyond the normalized
|
|
5075
5150
|
``CODEX_TEXT_CAP``. Truncation/``truncated`` is against ``_FULL_PAYLOAD_CEILING``
|
|
5076
|
-
(1,000,000 Python characters), the same ceiling the Claude payload path uses.
|
|
5151
|
+
(1,000,000 Python characters), the same ceiling the Claude payload path uses.
|
|
5152
|
+
|
|
5153
|
+
#968: select plain first, else zstd, and seek in decoded offsets. Streams are
|
|
5154
|
+
request-local and reusable; no decoded copies persist. Present unreadable
|
|
5155
|
+
containers raise CodexContainerReadError, never a false gone result."""
|
|
5077
5156
|
stored = conn.execute(
|
|
5078
5157
|
"SELECT payload_json FROM codex_conversation_events "
|
|
5079
5158
|
"WHERE source_path = ? AND line_offset = ?",
|
|
5080
5159
|
(row.source_path, row.line_offset)).fetchone()
|
|
5081
5160
|
if stored is None:
|
|
5082
5161
|
return None
|
|
5083
|
-
|
|
5084
|
-
|
|
5085
|
-
|
|
5086
|
-
|
|
5087
|
-
|
|
5088
|
-
|
|
5162
|
+
for attempt in range(2):
|
|
5163
|
+
selected = _select_codex_payload_container(row.source_path)
|
|
5164
|
+
if selected is None:
|
|
5165
|
+
return None
|
|
5166
|
+
# Validate exactly the descriptor we open, including after a compression
|
|
5167
|
+
# race. The caller has already checked the logical path and known root.
|
|
5168
|
+
root_path = _codex_source_root_path(conn, row.source_root_key)
|
|
5169
|
+
if not _within_root(str(selected.path), root_path):
|
|
5170
|
+
raise _CodexPayloadContainmentError()
|
|
5171
|
+
try:
|
|
5172
|
+
fh = None if streams is None else streams.get(str(selected.path))
|
|
5173
|
+
if fh is None:
|
|
5174
|
+
fh = _rc.open_decoded(selected)
|
|
5175
|
+
if streams is not None:
|
|
5176
|
+
streams[str(selected.path)] = fh
|
|
5177
|
+
try:
|
|
5178
|
+
fh.seek(row.line_offset)
|
|
5179
|
+
line = fh.readline()
|
|
5180
|
+
finally:
|
|
5181
|
+
if streams is None:
|
|
5182
|
+
fh.close()
|
|
5183
|
+
break
|
|
5184
|
+
except OSError as exc:
|
|
5185
|
+
# ContainerError subclasses OSError; preserve its approved leaf
|
|
5186
|
+
# category instead of flattening decoder/integrity errors.
|
|
5187
|
+
replacement = _select_codex_payload_container(row.source_path)
|
|
5188
|
+
if replacement is None:
|
|
5189
|
+
return None
|
|
5190
|
+
if replacement.path != selected.path and attempt == 0:
|
|
5191
|
+
continue # plain vanished during Codex's compression window
|
|
5192
|
+
category = exc.category if isinstance(exc, _rc.ContainerError) else "read-error"
|
|
5193
|
+
raise CodexContainerReadError(category, str(row.source_path)) from exc
|
|
5089
5194
|
try:
|
|
5090
5195
|
obj = json.loads(line.decode("utf-8"))
|
|
5091
5196
|
except (UnicodeDecodeError, ValueError, TypeError):
|
|
@@ -5172,7 +5277,9 @@ def read_codex_payload(
|
|
|
5172
5277
|
full bounded ``card``, and ``truncated`` reflects ``_FULL_PAYLOAD_CEILING``.
|
|
5173
5278
|
``gone`` (→ HTTP 410) means the physical
|
|
5174
5279
|
record moved/mutated; ``not_found`` (→ 404) means no such block, no output
|
|
5175
|
-
partner, or a containment miss (a read is never attempted outside the root).
|
|
5280
|
+
partner, or a containment miss (a read is never attempted outside the root).
|
|
5281
|
+
A present container that cannot be read raises CodexContainerReadError through
|
|
5282
|
+
the existing query-error envelope, including plain-file read failures."""
|
|
5176
5283
|
miss = {"status": "not_found", "block_key": block_key, "which": which}
|
|
5177
5284
|
if which not in ("call", "output", "event"):
|
|
5178
5285
|
return miss
|
|
@@ -5187,7 +5294,14 @@ def read_codex_payload(
|
|
|
5187
5294
|
root_path = _codex_source_root_path(conn, target.source_root_key)
|
|
5188
5295
|
if not _within_root(target.source_path, root_path):
|
|
5189
5296
|
return miss
|
|
5190
|
-
|
|
5297
|
+
streams: dict = {}
|
|
5298
|
+
try:
|
|
5299
|
+
outcome = _reread_codex_full_content(conn, target, streams=streams)
|
|
5300
|
+
except _CodexPayloadContainmentError:
|
|
5301
|
+
return miss
|
|
5302
|
+
finally:
|
|
5303
|
+
for stream in streams.values():
|
|
5304
|
+
stream.close()
|
|
5191
5305
|
if outcome is None:
|
|
5192
5306
|
return {"status": "gone", "block_key": block_key, "which": which}
|
|
5193
5307
|
content, truncated, card = outcome
|
|
@@ -43,15 +43,16 @@ All six §5.4 bullets are contract:
|
|
|
43
43
|
its first ingest attempt (so a discovered file is never lost even if its
|
|
44
44
|
first ingest is dirty/lock-contended and leaves no cursor row). A path is
|
|
45
45
|
retried while unclassified: with no committed cursor it is surfaced every
|
|
46
|
-
time it is visited; with a committed cursor it is surfaced only when its
|
|
47
|
-
|
|
48
|
-
any directory-mtime change). A path leaves the set when the driver ``reap``s
|
|
46
|
+
time it is visited; with a committed cursor it is surfaced only when its
|
|
47
|
+
signature advances beyond that cursor (an incomplete-``session_meta`` tail
|
|
48
|
+
completing without any directory-mtime change). A path leaves the set when the driver ``reap``s
|
|
49
49
|
it (it gained a thread row — child or non-child), when it vanishes, or on a
|
|
50
50
|
bounded per-candidate expiry (handed to the next full sync).
|
|
51
51
|
"""
|
|
52
52
|
from __future__ import annotations
|
|
53
53
|
|
|
54
54
|
import os
|
|
55
|
+
import _lib_codex_rollout_container as _rc
|
|
55
56
|
|
|
56
57
|
|
|
57
58
|
# ── production-default injected callables ─────────────────────────────────────
|
|
@@ -101,6 +102,48 @@ def default_file_size(filepath):
|
|
|
101
102
|
return st.st_size
|
|
102
103
|
|
|
103
104
|
|
|
105
|
+
def default_file_sig(filepath):
|
|
106
|
+
"""The selected container's live-tail signature (#968 spec 5.9).
|
|
107
|
+
|
|
108
|
+
Plain keeps today's size (as ``("plain", size)``); a compressed-only
|
|
109
|
+
rollout reports its container fingerprint instead of vanishing.
|
|
110
|
+
"""
|
|
111
|
+
return _rc.observe_watch_sig(filepath)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _as_sig(value):
|
|
115
|
+
if value is None or isinstance(value, tuple):
|
|
116
|
+
return value
|
|
117
|
+
return (_rc.PLAIN, int(value))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def codex_sig_advanced(observed, committed) -> bool:
|
|
121
|
+
"""Whether an observed signature is ahead of the committed one.
|
|
122
|
+
|
|
123
|
+
Plain grows (append-only, size decides); any other change of a compressed
|
|
124
|
+
container, or of the container kind, is a change. A bare int on either
|
|
125
|
+
side is a legacy plain size.
|
|
126
|
+
"""
|
|
127
|
+
obs, com = _as_sig(observed), _as_sig(committed)
|
|
128
|
+
if obs is None:
|
|
129
|
+
return False
|
|
130
|
+
if com is None:
|
|
131
|
+
return True
|
|
132
|
+
if obs[0] == com[0] == _rc.PLAIN:
|
|
133
|
+
return obs[1] > com[1]
|
|
134
|
+
return obs != com
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def committed_watch_sig(kind, size_bytes, container_size_bytes, mtime_ns,
|
|
138
|
+
device, inode):
|
|
139
|
+
"""The signature a cursor row's ACCEPTED container would observe: the
|
|
140
|
+
leaf's ``ContainerFingerprint.watch_sig`` shape, device included."""
|
|
141
|
+
if kind in (None, _rc.PLAIN):
|
|
142
|
+
return (_rc.PLAIN, int(size_bytes or 0))
|
|
143
|
+
return (_rc.ZSTD, int(container_size_bytes or 0), int(mtime_ns or 0),
|
|
144
|
+
int(device or 0), int(inode or 0))
|
|
145
|
+
|
|
146
|
+
|
|
104
147
|
_SENTINEL = object()
|
|
105
148
|
|
|
106
149
|
|
|
@@ -118,7 +161,7 @@ class CodexChildFrontier:
|
|
|
118
161
|
pending_expiry_cycles: int = 50,
|
|
119
162
|
scandir_fn=default_scandir,
|
|
120
163
|
dir_mtime_fn=default_dir_mtime,
|
|
121
|
-
file_size_fn=
|
|
164
|
+
file_size_fn=default_file_sig,
|
|
122
165
|
clock_fn=None,
|
|
123
166
|
):
|
|
124
167
|
self._walk_root = str(walk_root)
|
|
@@ -175,7 +218,7 @@ class CodexChildFrontier:
|
|
|
175
218
|
|
|
176
219
|
``known_paths`` — every path already tracked in ``codex_session_files``,
|
|
177
220
|
so a newly enumerated file already known to some table is not
|
|
178
|
-
re-registered. ``committed_sizes`` — ``{path:
|
|
221
|
+
re-registered. ``committed_sizes`` — ``{path: signature}`` committed
|
|
179
222
|
cursors, for the pending-growth check.
|
|
180
223
|
|
|
181
224
|
Returns the set of candidate paths to targeted-ingest this cycle (new
|
|
@@ -251,8 +294,8 @@ class CodexChildFrontier:
|
|
|
251
294
|
mtime as the dir's baseline (so a create landing during the enumeration
|
|
252
295
|
is re-examined next rotation), skips symlinked descendants (the
|
|
253
296
|
``follow_symlinks=False`` boundary), adds real subdirectories to the
|
|
254
|
-
frontier, and registers brand-new
|
|
255
|
-
their first ingest — surfacing each for that first ingest attempt in the
|
|
297
|
+
frontier, and registers brand-new logical rollout files in ``pending``
|
|
298
|
+
BEFORE their first ingest — surfacing each for that first ingest attempt in the
|
|
256
299
|
SAME cycle it is discovered (the pending sweep then owns the retries).
|
|
257
300
|
Returns True iff it consumed a filesystem op."""
|
|
258
301
|
if not self._enum_queue:
|
|
@@ -267,10 +310,12 @@ class CodexChildFrontier:
|
|
|
267
310
|
if path not in self._dir_set:
|
|
268
311
|
self._dir_set.add(path)
|
|
269
312
|
self._dirs.append(path) # new subdir joins frontier
|
|
270
|
-
elif
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
313
|
+
elif _rc.is_container_name(name):
|
|
314
|
+
# Both spellings enumerate one logical identity.
|
|
315
|
+
logical = os.fspath(_rc.logical_path(path))
|
|
316
|
+
if logical not in self._known and logical not in self._pending:
|
|
317
|
+
self._pending[logical] = 0 # register BEFORE first ingest
|
|
318
|
+
ingest.add(logical) # first ingest attempt now
|
|
274
319
|
return True
|
|
275
320
|
|
|
276
321
|
def _op_pending(self, ingest: set) -> bool:
|
|
@@ -286,12 +331,12 @@ class CodexChildFrontier:
|
|
|
286
331
|
p = keys[idx]
|
|
287
332
|
if p not in self._pending: # reaped/dropped mid-cycle
|
|
288
333
|
continue
|
|
289
|
-
|
|
290
|
-
if
|
|
334
|
+
sig = self._file_size(p) # 1 signature probe
|
|
335
|
+
if sig is None:
|
|
291
336
|
self._pending.pop(p, None) # vanished → drop
|
|
292
337
|
else:
|
|
293
338
|
committed = self._committed.get(p)
|
|
294
|
-
if
|
|
339
|
+
if codex_sig_advanced(sig, committed):
|
|
295
340
|
ingest.add(p)
|
|
296
341
|
return True
|
|
297
342
|
return False
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Waiting requests and the drain-admission gate (#968 spec 5.7, 5.12).
|
|
2
|
+
|
|
3
|
+
A Codex hook never waits for a lock. When it meets cache contention it
|
|
4
|
+
publishes a uniquely named, immutable request record under the data
|
|
5
|
+
directory (outside every SQLite store, because a contended hook cannot write
|
|
6
|
+
cache.db). Drain workers — the prune executor and the container-verification
|
|
7
|
+
executor — admit one identity at a time through ONE global gate and yield
|
|
8
|
+
while any request is fresh:
|
|
9
|
+
|
|
10
|
+
1. check for fresh requests;
|
|
11
|
+
2. acquire the drain gate (non-blocking), then check again;
|
|
12
|
+
3. try the store's complete lock domain without blocking, then check again;
|
|
13
|
+
4. a fresh request at any check releases everything acquired and yields;
|
|
14
|
+
5. otherwise the caller executes one identity, commits, releases every store
|
|
15
|
+
lock and the gate (``finish``), and starts again at step 1.
|
|
16
|
+
|
|
17
|
+
Requests end only by expiry: 60 s after publication on a cross-process
|
|
18
|
+
monotonic clock. A worker observing a request, or restarting, never renews it;
|
|
19
|
+
a wall-clock jump cannot extend it (the wall age only shortens freshness, and
|
|
20
|
+
a NEGATIVE wall age — an mtime ahead of the wall clock after a backward step —
|
|
21
|
+
counts as fresh, bounded by the monotonic lifetime); a malformed record or one
|
|
22
|
+
from an incompatible clock is expired and removed. Stdlib only; no cctally
|
|
23
|
+
imports.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import fcntl
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import pathlib
|
|
31
|
+
import time
|
|
32
|
+
import uuid
|
|
33
|
+
from typing import Callable, NamedTuple, Optional
|
|
34
|
+
|
|
35
|
+
REQUEST_DIR_NAME = "codex-drain-requests"
|
|
36
|
+
REQUEST_TTL_NS = 60_000_000_000
|
|
37
|
+
WALL_GUARD_SECONDS = 120.0
|
|
38
|
+
GATE_FILE_NAME = "codex-drain-admission.lock"
|
|
39
|
+
POLL_SECONDS = 5.0
|
|
40
|
+
_RECORD_VERSION = 1
|
|
41
|
+
|
|
42
|
+
# Module attributes so tests can drive a deterministic schedule.
|
|
43
|
+
_monotonic_ns: Callable[[], int] = time.monotonic_ns
|
|
44
|
+
_sleep: Callable[[float], None] = time.sleep
|
|
45
|
+
#: Post-release seam: called by ``finish`` after the store locks AND the gate
|
|
46
|
+
#: are released. Production leaves it None; the fairness test synchronizes on
|
|
47
|
+
#: it instead of sleeping (review P2-7).
|
|
48
|
+
_after_finish: Optional[Callable[[], None]] = None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _request_dir(app_dir) -> pathlib.Path:
|
|
52
|
+
return pathlib.Path(app_dir) / REQUEST_DIR_NAME
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def publish_waiting_request(app_dir, *, root_keys) -> "str | None":
|
|
56
|
+
"""Atomically publish one immutable request; return its id or None.
|
|
57
|
+
|
|
58
|
+
``None`` is a protocol failure the hook records; it is never fairness
|
|
59
|
+
evidence and never raises.
|
|
60
|
+
"""
|
|
61
|
+
try:
|
|
62
|
+
directory = _request_dir(app_dir)
|
|
63
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
64
|
+
mono = int(_monotonic_ns())
|
|
65
|
+
request_id = f"{mono}-{os.getpid()}-{uuid.uuid4().hex}"
|
|
66
|
+
final = directory / f"req-{request_id}.json"
|
|
67
|
+
tmp = directory / f".tmp-{request_id}"
|
|
68
|
+
payload = json.dumps({
|
|
69
|
+
"v": _RECORD_VERSION, "id": request_id,
|
|
70
|
+
"roots": sorted(str(k) for k in root_keys), "mono_ns": mono,
|
|
71
|
+
}, separators=(",", ":"))
|
|
72
|
+
with open(tmp, "x", encoding="utf-8") as fh:
|
|
73
|
+
fh.write(payload)
|
|
74
|
+
fh.flush()
|
|
75
|
+
os.fsync(fh.fileno())
|
|
76
|
+
os.rename(tmp, final)
|
|
77
|
+
return request_id
|
|
78
|
+
except OSError:
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _is_fresh(path: pathlib.Path, now_mono: int, now_wall: float) -> bool:
|
|
83
|
+
try:
|
|
84
|
+
record = json.loads(path.read_text(encoding="utf-8"))
|
|
85
|
+
mono = int(record["mono_ns"])
|
|
86
|
+
if record.get("v") != _RECORD_VERSION:
|
|
87
|
+
return False
|
|
88
|
+
wall_age = now_wall - path.stat().st_mtime
|
|
89
|
+
except (OSError, ValueError, TypeError, KeyError):
|
|
90
|
+
return False
|
|
91
|
+
age = now_mono - mono
|
|
92
|
+
# A negative monotonic age is an incompatible clock: expired. A negative
|
|
93
|
+
# WALL age is only an mtime ahead of the wall clock, which is not evidence
|
|
94
|
+
# of staleness: it stays fresh, and the monotonic bound still ends it
|
|
95
|
+
# within the single 60 s lifetime (review P3-5).
|
|
96
|
+
return 0 <= age < REQUEST_TTL_NS and wall_age < WALL_GUARD_SECONDS
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def fresh_requests(app_dir) -> "list[str]":
|
|
100
|
+
"""Ids of fresh requests; expired or malformed records are removed one by one."""
|
|
101
|
+
directory = _request_dir(app_dir)
|
|
102
|
+
try:
|
|
103
|
+
entries = sorted(directory.iterdir())
|
|
104
|
+
except OSError:
|
|
105
|
+
return []
|
|
106
|
+
now_mono = int(_monotonic_ns())
|
|
107
|
+
now_wall = time.time()
|
|
108
|
+
fresh: list[str] = []
|
|
109
|
+
for path in entries:
|
|
110
|
+
name = path.name
|
|
111
|
+
if name.startswith(".tmp-"):
|
|
112
|
+
try:
|
|
113
|
+
if now_wall - path.stat().st_mtime >= WALL_GUARD_SECONDS:
|
|
114
|
+
path.unlink()
|
|
115
|
+
except OSError:
|
|
116
|
+
pass
|
|
117
|
+
continue
|
|
118
|
+
if not (name.startswith("req-") and name.endswith(".json")):
|
|
119
|
+
continue
|
|
120
|
+
if _is_fresh(path, now_mono, now_wall):
|
|
121
|
+
fresh.append(name[len("req-"):-len(".json")])
|
|
122
|
+
else:
|
|
123
|
+
try:
|
|
124
|
+
path.unlink()
|
|
125
|
+
except OSError:
|
|
126
|
+
pass
|
|
127
|
+
return fresh
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class DrainGate:
|
|
131
|
+
"""The one global, non-blocking drain-admission flock."""
|
|
132
|
+
|
|
133
|
+
def __init__(self, app_dir):
|
|
134
|
+
self._path = pathlib.Path(app_dir) / GATE_FILE_NAME
|
|
135
|
+
self._fd: "int | None" = None
|
|
136
|
+
|
|
137
|
+
def try_acquire(self) -> bool:
|
|
138
|
+
if self._fd is not None:
|
|
139
|
+
return True
|
|
140
|
+
self._path.parent.mkdir(parents=True, exist_ok=True)
|
|
141
|
+
fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
|
|
142
|
+
try:
|
|
143
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
144
|
+
except OSError:
|
|
145
|
+
os.close(fd)
|
|
146
|
+
return False
|
|
147
|
+
self._fd = fd
|
|
148
|
+
return True
|
|
149
|
+
|
|
150
|
+
def release(self) -> None:
|
|
151
|
+
if self._fd is None:
|
|
152
|
+
return
|
|
153
|
+
try:
|
|
154
|
+
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
|
155
|
+
finally:
|
|
156
|
+
os.close(self._fd)
|
|
157
|
+
self._fd = None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class Admission(NamedTuple):
|
|
161
|
+
admitted: bool
|
|
162
|
+
store_locks: object
|
|
163
|
+
gate: "DrainGate | None"
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def admit(app_dir, try_store_locks: Callable[[], object],
|
|
167
|
+
release_store_locks: Callable[[object], None], *,
|
|
168
|
+
on_check: Optional[Callable[[str], None]] = None) -> Admission:
|
|
169
|
+
"""Steps 1-4 of the protocol. ``try_store_locks`` must never block and
|
|
170
|
+
returns a token, or None on contention."""
|
|
171
|
+
def check(stage: str) -> bool:
|
|
172
|
+
if on_check is not None:
|
|
173
|
+
on_check(stage)
|
|
174
|
+
return bool(fresh_requests(app_dir))
|
|
175
|
+
|
|
176
|
+
if check("before-gate"):
|
|
177
|
+
return Admission(False, None, None)
|
|
178
|
+
gate = DrainGate(app_dir)
|
|
179
|
+
if not gate.try_acquire():
|
|
180
|
+
return Admission(False, None, None)
|
|
181
|
+
if check("after-gate"):
|
|
182
|
+
gate.release()
|
|
183
|
+
return Admission(False, None, None)
|
|
184
|
+
token = try_store_locks()
|
|
185
|
+
if token is None:
|
|
186
|
+
gate.release()
|
|
187
|
+
return Admission(False, None, None)
|
|
188
|
+
if check("after-store-locks"):
|
|
189
|
+
release_store_locks(token)
|
|
190
|
+
gate.release()
|
|
191
|
+
return Admission(False, None, None)
|
|
192
|
+
return Admission(True, token, gate)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def finish(admission: Admission,
|
|
196
|
+
release_store_locks: Callable[[object], None]) -> None:
|
|
197
|
+
"""Step 5: release every store lock, then the gate."""
|
|
198
|
+
try:
|
|
199
|
+
if admission.store_locks is not None:
|
|
200
|
+
release_store_locks(admission.store_locks)
|
|
201
|
+
finally:
|
|
202
|
+
if admission.gate is not None:
|
|
203
|
+
admission.gate.release()
|
|
204
|
+
if _after_finish is not None:
|
|
205
|
+
_after_finish()
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def wait_until_no_fresh_requests(
|
|
209
|
+
app_dir, *, should_continue: Optional[Callable[[], bool]] = None) -> None:
|
|
210
|
+
"""Sleep in bounded steps until no request is fresh (requests expire)."""
|
|
211
|
+
while fresh_requests(app_dir):
|
|
212
|
+
if should_continue is not None and not should_continue():
|
|
213
|
+
return
|
|
214
|
+
_sleep(POLL_SECONDS)
|
|
@@ -26,6 +26,8 @@ import sqlite3
|
|
|
26
26
|
import threading
|
|
27
27
|
from collections import OrderedDict
|
|
28
28
|
|
|
29
|
+
from _lib_codex_publication_barrier import LAST_GOOD_THREADS_VIEW
|
|
30
|
+
|
|
29
31
|
|
|
30
32
|
#: The physical name of the raw threads table. ``resolve_codex_threads_table``
|
|
31
33
|
#: returns it qualified by whichever schema actually holds it.
|
|
@@ -185,6 +187,8 @@ def _schema_holds_threads_table(conn: sqlite3.Connection, schema: str) -> bool:
|
|
|
185
187
|
def resolve_codex_threads_table(conn: sqlite3.Connection) -> str:
|
|
186
188
|
"""The RAW threads table reference for this connection.
|
|
187
189
|
|
|
190
|
+
A conversations connection carrying the #968 last-good view resolves to it.
|
|
191
|
+
|
|
188
192
|
A bare cache connection (``open_cache_db()``) holds the table in its own
|
|
189
193
|
``main`` schema; a conversations connection
|
|
190
194
|
(``open_conversations_db``/``open_conversations_db_readonly``) reaches it
|
|
@@ -222,8 +226,21 @@ def resolve_codex_threads_table(conn: sqlite3.Connection) -> str:
|
|
|
222
226
|
_threads_table_resolution_cache.move_to_end(conn)
|
|
223
227
|
return cached
|
|
224
228
|
|
|
229
|
+
# One metadata SELECT preserves last-good priority without adding a
|
|
230
|
+
# fifth query to the bounded accounting/metadata-health read contracts.
|
|
231
|
+
schema = conn.execute(
|
|
232
|
+
"SELECT CASE "
|
|
233
|
+
"WHEN EXISTS (SELECT 1 FROM sqlite_temp_master "
|
|
234
|
+
"WHERE type='view' AND name=?) THEN 'temp' "
|
|
235
|
+
"WHEN EXISTS (SELECT 1 FROM main.sqlite_master "
|
|
236
|
+
"WHERE type='table' AND name=?) THEN 'main' END",
|
|
237
|
+
(LAST_GOOD_THREADS_VIEW, CODEX_THREADS_TABLE),
|
|
238
|
+
).fetchone()[0]
|
|
225
239
|
resolved = None
|
|
226
|
-
if
|
|
240
|
+
if schema == "temp":
|
|
241
|
+
# #968 spec 5.11: unpublished metadata never overrides last-good.
|
|
242
|
+
resolved = f"temp.{LAST_GOOD_THREADS_VIEW}"
|
|
243
|
+
elif schema == "main":
|
|
227
244
|
resolved = f"main.{CODEX_THREADS_TABLE}"
|
|
228
245
|
else:
|
|
229
246
|
attached = tuple(conn.execute("PRAGMA database_list"))
|