cctally 1.115.0 → 1.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/bin/_cctally_cache.py +85 -4
- package/bin/_cctally_completion.py +80 -12
- package/bin/_cctally_core.py +106 -9
- package/bin/_cctally_dashboard.py +6 -9
- package/bin/_cctally_dashboard_envelope.py +26 -32
- package/bin/_cctally_db.py +85 -4
- package/bin/_cctally_doctor.py +38 -5
- package/bin/_cctally_five_hour.py +83 -1017
- package/bin/_cctally_journal.py +1307 -180
- package/bin/_cctally_journal_repair.py +9 -7
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_quarantine.py +14 -4
- package/bin/_cctally_quota.py +70 -8
- package/bin/_cctally_quota_model.py +35 -4
- package/bin/_cctally_record.py +414 -123
- package/bin/_cctally_rederive.py +632 -235
- package/bin/_cctally_refresh.py +29 -0
- package/bin/_cctally_statusline.py +984 -47
- package/bin/_cctally_store.py +972 -58
- package/bin/_cctally_tui.py +62 -7
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_protocol.py +201 -18
- package/bin/_lib_clone_copy.py +46 -11
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +273 -52
- package/bin/_lib_doctor.py +302 -1
- package/bin/_lib_journal.py +145 -25
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +3 -2
- package/bin/_lib_quota_model.py +96 -6
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +299 -3
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +65 -5
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +6 -11
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +7 -2
- package/bin/_lib_merge_gate.py +0 -452
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
package/bin/_cctally_tui.py
CHANGED
|
@@ -5919,7 +5919,8 @@ def _tui_precompute_doctor_payload(
|
|
|
5919
5919
|
"""Precompute the dashboard envelope's small doctor block, via the
|
|
5920
5920
|
short-TTL memo (#268 M4, spec §6).
|
|
5921
5921
|
|
|
5922
|
-
Returns the SAME ``{severity, counts, generated_at, fingerprint
|
|
5922
|
+
Returns the SAME ``{severity, counts, generated_at, fingerprint,
|
|
5923
|
+
rebuild_worker}`` dict (``_lib_doctor.envelope_block``)
|
|
5923
5924
|
``snapshot_to_envelope`` used to build inline (or a synthetic FAIL block
|
|
5924
5925
|
with ``_error`` on a doctor gather/checks failure), so moving the
|
|
5925
5926
|
computation onto the snapshot is byte-identical for a given
|
|
@@ -5938,12 +5939,7 @@ def _tui_precompute_doctor_payload(
|
|
|
5938
5939
|
now_utc=now, runtime_bind=bind)
|
|
5939
5940
|
with _perf.phase("doctor.checks"):
|
|
5940
5941
|
_doc_report = _ld.run_checks(_doc_state)
|
|
5941
|
-
return
|
|
5942
|
-
"severity": _doc_report.overall_severity,
|
|
5943
|
-
"counts": dict(_doc_report.counts),
|
|
5944
|
-
"generated_at": _ld._iso_z(_doc_report.generated_at),
|
|
5945
|
-
"fingerprint": _ld.fingerprint(_doc_report),
|
|
5946
|
-
}
|
|
5942
|
+
return _ld.envelope_block(_doc_report)
|
|
5947
5943
|
except Exception as exc: # noqa: BLE001 — never crash the rebuild
|
|
5948
5944
|
# Mirror `snapshot_to_envelope`'s inline FAIL fallback (dashboard
|
|
5949
5945
|
# `_iso_z`: strftime, no microseconds) so a doctor failure serves
|
|
@@ -5955,6 +5951,7 @@ def _tui_precompute_doctor_payload(
|
|
|
5955
5951
|
"%Y-%m-%dT%H:%M:%SZ"
|
|
5956
5952
|
),
|
|
5957
5953
|
"fingerprint": "sha1:" + ("0" * 40),
|
|
5954
|
+
"rebuild_worker": None,
|
|
5958
5955
|
"_error": f"{type(exc).__name__}: {exc}",
|
|
5959
5956
|
}
|
|
5960
5957
|
|
|
@@ -5963,6 +5960,51 @@ def _tui_precompute_doctor_payload(
|
|
|
5963
5960
|
)
|
|
5964
5961
|
|
|
5965
5962
|
|
|
5963
|
+
def _deferred_doctor_placeholder(now_utc: dt.datetime) -> dict:
|
|
5964
|
+
"""The schema-valid, privacy-safe doctor block a deferred frame carries
|
|
5965
|
+
while the real doctor gather cannot run (it opens stats repeatedly); the
|
|
5966
|
+
first successful snapshot replaces it with a real doctor payload."""
|
|
5967
|
+
return {
|
|
5968
|
+
"severity": "warn",
|
|
5969
|
+
"counts": {"ok": 0, "warn": 1, "fail": 0},
|
|
5970
|
+
"generated_at": now_utc.astimezone(dt.timezone.utc).strftime(
|
|
5971
|
+
"%Y-%m-%dT%H:%M:%SZ"
|
|
5972
|
+
),
|
|
5973
|
+
"fingerprint": "sha1:" + ("0" * 40),
|
|
5974
|
+
"rebuild_worker": None,
|
|
5975
|
+
}
|
|
5976
|
+
|
|
5977
|
+
|
|
5978
|
+
def _tui_rebuild_worker_value(now_utc: dt.datetime, fallback=None):
|
|
5979
|
+
"""#997 D10: the doctor block's ``rebuild_worker`` value for a deferred
|
|
5980
|
+
frame (``{severity, summary}`` or ``None``), via a short-TTL memo.
|
|
5981
|
+
|
|
5982
|
+
While an epoch rebuild is pending or running every live stats open
|
|
5983
|
+
defers, so neither the deferred startup frame nor a deferred tick can run
|
|
5984
|
+
the doctor gather; without this the chip would show only its placeholder
|
|
5985
|
+
for the whole run. The value comes from the worker snapshots alone
|
|
5986
|
+
(`_cctally_store.rebuild_worker_snapshot`: file reads, the stats.db
|
|
5987
|
+
header bytes and a `ps` probe; it never opens stats.db and takes no
|
|
5988
|
+
cctally lock, because the detached worker owns maintenance-exclusive)
|
|
5989
|
+
through the same `_lib_doctor` assessment as the `db.rebuild_worker`
|
|
5990
|
+
check. Never raises: a failure returns ``fallback``.
|
|
5991
|
+
"""
|
|
5992
|
+
try:
|
|
5993
|
+
c = _cctally()
|
|
5994
|
+
sc = c._load_sibling("_lib_snapshot_cache")
|
|
5995
|
+
|
|
5996
|
+
def _compute(now: dt.datetime):
|
|
5997
|
+
_ld = c._load_sibling("_lib_doctor")
|
|
5998
|
+
_cd = c._load_sibling("_cctally_doctor")
|
|
5999
|
+
return _ld.rebuild_worker_envelope_value(
|
|
6000
|
+
_cd._gather_rebuild_workers(), now)
|
|
6001
|
+
|
|
6002
|
+
return sc.rebuild_worker_memo(
|
|
6003
|
+
now_utc, ttl_s=sc.REBUILD_WORKER_MEMO_TTL_S, compute=_compute)
|
|
6004
|
+
except Exception: # noqa: BLE001 — a frame must never fail on this
|
|
6005
|
+
return fallback
|
|
6006
|
+
|
|
6007
|
+
|
|
5966
6008
|
def _tui_precompute_envelope_config(raw_config: dict) -> dict:
|
|
5967
6009
|
"""Precompute the config / update-state reads `snapshot_to_envelope`
|
|
5968
6010
|
needs (#268 M4, spec §6).
|
|
@@ -9201,8 +9243,21 @@ def _make_run_sync_now_locked(*, ref, hub, pinned_now, display_tz_pref_override,
|
|
|
9201
9243
|
# an index that could not be read, and reporting that as
|
|
9202
9244
|
# non-corruption makes the envelope name the wrong fault.
|
|
9203
9245
|
prev = ref.get()
|
|
9246
|
+
# #997 D10: the doctor gather cannot run during the deferral, so
|
|
9247
|
+
# refresh only the rebuild-worker aggregate the chip leads with.
|
|
9248
|
+
deferred_now = pinned_now or dt.datetime.now(dt.timezone.utc)
|
|
9249
|
+
prev_doctor = getattr(prev, "doctor_payload", None)
|
|
9250
|
+
base_doctor = (prev_doctor if isinstance(prev_doctor, dict)
|
|
9251
|
+
else _deferred_doctor_placeholder(deferred_now))
|
|
9252
|
+
doctor_payload = {
|
|
9253
|
+
**base_doctor,
|
|
9254
|
+
"rebuild_worker": _tui_rebuild_worker_value(
|
|
9255
|
+
deferred_now,
|
|
9256
|
+
fallback=base_doctor.get("rebuild_worker")),
|
|
9257
|
+
}
|
|
9204
9258
|
pending = dataclasses.replace(
|
|
9205
9259
|
prev,
|
|
9260
|
+
doctor_payload=doctor_payload,
|
|
9206
9261
|
last_sync_error=f"stats-open: {exc}",
|
|
9207
9262
|
sync_failures=(
|
|
9208
9263
|
SyncFailureAttribution(
|
package/bin/_cctally_update.py
CHANGED
|
@@ -1475,6 +1475,31 @@ def _spawn_detached(command: str) -> bool:
|
|
|
1475
1475
|
return False
|
|
1476
1476
|
|
|
1477
1477
|
|
|
1478
|
+
def _spawn_detached_with_pid(
|
|
1479
|
+
command: str, *, env: "dict | None" = None,
|
|
1480
|
+
) -> "int | None":
|
|
1481
|
+
"""`_spawn_detached`'s launch shape, returning the child's pid.
|
|
1482
|
+
|
|
1483
|
+
The detached stats rebuild workers (#997 D8) record each launch's pid and
|
|
1484
|
+
start identity and hand the child its launch id through ``env``, so they
|
|
1485
|
+
need the pid that `_spawn_detached`'s bool contract (kept for its other
|
|
1486
|
+
callers) does not carry. ``None`` when the spawn raised.
|
|
1487
|
+
"""
|
|
1488
|
+
try:
|
|
1489
|
+
proc = subprocess.Popen(
|
|
1490
|
+
[sys.executable, os.path.realpath(sys.argv[0]), command],
|
|
1491
|
+
stdin=subprocess.DEVNULL,
|
|
1492
|
+
stdout=subprocess.DEVNULL,
|
|
1493
|
+
stderr=subprocess.DEVNULL,
|
|
1494
|
+
start_new_session=True,
|
|
1495
|
+
close_fds=True,
|
|
1496
|
+
env=env,
|
|
1497
|
+
)
|
|
1498
|
+
except Exception:
|
|
1499
|
+
return None
|
|
1500
|
+
return proc.pid
|
|
1501
|
+
|
|
1502
|
+
|
|
1478
1503
|
def _spawn_background_update_check() -> None:
|
|
1479
1504
|
"""Fire-and-forget the hidden ``_update-check`` worker (spec §3.6)."""
|
|
1480
1505
|
_spawn_detached("_update-check")
|
|
@@ -24,6 +24,15 @@ post-boundary record without the current writer generation is admitted only
|
|
|
24
24
|
when an explicit field its real producer writes proves it Codex-only. An
|
|
25
25
|
unknown or unclassifiable record is excluded, never admitted by omission.
|
|
26
26
|
|
|
27
|
+
#797/#823/#840 (spec section 4.6, B1-B3): revision 2 of the fence. ``wpg``
|
|
28
|
+
keeps its value (the previous release's exact-equality ``is_stamped`` must
|
|
29
|
+
keep admitting what the current binary writes), and a second reserved
|
|
30
|
+
top-level field ``wpr`` carries the integer protocol revision. A durable
|
|
31
|
+
``account_bound_revision`` op records a second physical boundary with the same
|
|
32
|
+
codec; after it, a Claude-scoped record without ``wpr == 2`` is excluded
|
|
33
|
+
exactly as an unstamped record is after the original cutover. The revision
|
|
34
|
+
boundary never re-admits a record the original rule excludes.
|
|
35
|
+
|
|
27
36
|
Pure: no I/O, no clock, no SQLite.
|
|
28
37
|
"""
|
|
29
38
|
from __future__ import annotations
|
|
@@ -39,6 +48,12 @@ WRITER_GENERATION_KEY = "wpg"
|
|
|
39
48
|
CUTOVER_OP_KIND = "account_bound_cutover"
|
|
40
49
|
CUTOVER_OP_ID = "account-bound-cutover-v1"
|
|
41
50
|
LEGACY_SPOOL_ACTIVITY_KIND = "legacy_spool_activity"
|
|
51
|
+
#: #797/#823/#840 (spec 4.6 B1, B2): the protocol revision stamp and the
|
|
52
|
+
#: durable op that records where revision 2 begins.
|
|
53
|
+
REVISION_KEY = "wpr"
|
|
54
|
+
PROTOCOL_REVISION = 2
|
|
55
|
+
REVISION_OP_KIND = "account_bound_revision"
|
|
56
|
+
REVISION_OP_ID = "account-bound-revision-2"
|
|
42
57
|
#: Spec revision 5 (section 7, operator decision on Codex design-fix-round-5
|
|
43
58
|
#: Q4): the `legacy_spool_activity` reason of an incomplete inventory. Such an
|
|
44
59
|
#: op records typed failure evidence and no digest, so it anchors no baseline.
|
|
@@ -59,6 +74,11 @@ INVENTORY_OPERATIONS = ("list", "stat", "read")
|
|
|
59
74
|
#: previous-release validators require exact keys (spec 5.1). The previous
|
|
60
75
|
#: release ignores an unknown op kind, so the current binary emits only these.
|
|
61
76
|
PROTOCOL_RESOLUTION_V2_KIND = "journal_protocol_resolution_v2"
|
|
77
|
+
#: #797/#823/#840 (spec 4.6 B5): the previous release's strict validator of
|
|
78
|
+
#: the v2 acknowledgement rejects the extra ``wpr`` field, so the current
|
|
79
|
+
#: binary emits repair acknowledgements under this successor kind, which the
|
|
80
|
+
#: previous release ignores as an unknown op.
|
|
81
|
+
PROTOCOL_RESOLUTION_V3_KIND = "journal_protocol_resolution_v3"
|
|
62
82
|
WEEKLY_OBSERVATION_DECISION_V2_KIND = "claude_weekly_observation_decision_v2"
|
|
63
83
|
|
|
64
84
|
#: Admission families. ``codex`` is the only family admitted without a stamp.
|
|
@@ -73,7 +93,9 @@ FAMILIES = (
|
|
|
73
93
|
)
|
|
74
94
|
|
|
75
95
|
ADMISSION_REASONS = ("no-cutover", "historical", "stamped", "codex-only",
|
|
76
|
-
"excluded")
|
|
96
|
+
"excluded", "revision-unstamped")
|
|
97
|
+
#: The reasons that exclude a record.
|
|
98
|
+
EXCLUDED_REASONS = frozenset(("excluded", "revision-unstamped"))
|
|
77
99
|
|
|
78
100
|
_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
79
101
|
|
|
@@ -87,9 +109,11 @@ _CODEX_OP_KINDS = frozenset((
|
|
|
87
109
|
))
|
|
88
110
|
_ADMIN_OP_KINDS = frozenset((
|
|
89
111
|
"journal_protocol_resolution", "claude_weekly_observation_decision",
|
|
90
|
-
PROTOCOL_RESOLUTION_V2_KIND,
|
|
112
|
+
PROTOCOL_RESOLUTION_V2_KIND, PROTOCOL_RESOLUTION_V3_KIND,
|
|
113
|
+
WEEKLY_OBSERVATION_DECISION_V2_KIND,
|
|
91
114
|
))
|
|
92
|
-
_PROTOCOL_OP_KINDS = frozenset((CUTOVER_OP_KIND, LEGACY_SPOOL_ACTIVITY_KIND
|
|
115
|
+
_PROTOCOL_OP_KINDS = frozenset((CUTOVER_OP_KIND, LEGACY_SPOOL_ACTIVITY_KIND,
|
|
116
|
+
REVISION_OP_KIND))
|
|
93
117
|
_ADMIN_RECORD_TYPES = frozenset(("correction", "correction_batch"))
|
|
94
118
|
_CLAUDE_EVT_KINDS = frozenset((
|
|
95
119
|
"snapshot_accept", "weekly_cost_snapshot", "week_reset", "five_hour_credit",
|
|
@@ -133,6 +157,43 @@ class CutoverBoundary:
|
|
|
133
157
|
frozen_extents: Mapping[str, int]
|
|
134
158
|
legacy_spool_baseline_digest: "str | None"
|
|
135
159
|
legacy_spool_inventory_failure: "Mapping[str, object] | None" = None
|
|
160
|
+
#: #797/#823/#840 (spec 4.6 B3): the revision-2 boundary a lookup found
|
|
161
|
+
#: after this cutover, carried beside it so every admission consumer
|
|
162
|
+
#: applies both fences through the one boundary object it already
|
|
163
|
+
#: threads. Not part of the cutover op's codec, and not part of equality:
|
|
164
|
+
#: two lookups of one cutover op name the same original boundary, and
|
|
165
|
+
#: the admission summary records the revision separately.
|
|
166
|
+
revision: "RevisionBoundary | None" = dataclasses.field(
|
|
167
|
+
default=None, compare=False)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@dataclasses.dataclass(frozen=True)
|
|
171
|
+
class RevisionBoundary:
|
|
172
|
+
"""The revision-2 boundary recorded by the ``account_bound_revision`` op
|
|
173
|
+
(spec 4.6 B2): the same physical-position codec as `CutoverBoundary`,
|
|
174
|
+
plus the original cutover it extends (``cutover_segment`` and
|
|
175
|
+
``cutover_offset``, the coordinate of the cutover op).
|
|
176
|
+
|
|
177
|
+
``legacy_spool_baseline_digest`` is the EXPANDED baseline digest — the
|
|
178
|
+
pre-#825 legacy paths and the retired v1 directory's coordination state
|
|
179
|
+
(spec 4.6 B4) — or None when the establishment could not complete that
|
|
180
|
+
inventory and records ``legacy_spool_inventory_failure`` instead, exactly
|
|
181
|
+
as a cutover does (#825 spec revision 5).
|
|
182
|
+
|
|
183
|
+
``continued_inventory_episode`` is the coordinate of the uncertainty
|
|
184
|
+
record whose failure episode was still open when an incomplete revision
|
|
185
|
+
inventory was recorded: the failure belongs to that episode, which
|
|
186
|
+
already has its trigger, so the revision op raises none of its own (one
|
|
187
|
+
trigger per failure episode, #825 spec revision 5). None otherwise."""
|
|
188
|
+
|
|
189
|
+
segment: str
|
|
190
|
+
end_offset: int
|
|
191
|
+
frozen_extents: Mapping[str, int]
|
|
192
|
+
legacy_spool_baseline_digest: "str | None"
|
|
193
|
+
cutover_segment: str
|
|
194
|
+
cutover_offset: int
|
|
195
|
+
legacy_spool_inventory_failure: "Mapping[str, object] | None" = None
|
|
196
|
+
continued_inventory_episode: "tuple[str, int] | None" = None
|
|
136
197
|
|
|
137
198
|
|
|
138
199
|
@dataclasses.dataclass(frozen=True)
|
|
@@ -148,17 +209,23 @@ class Admission:
|
|
|
148
209
|
|
|
149
210
|
def stamp(record: dict) -> dict:
|
|
150
211
|
"""Return a shallow copy of ``record`` carrying the current writer
|
|
151
|
-
generation
|
|
152
|
-
|
|
153
|
-
|
|
212
|
+
generation and protocol revision (spec 4.6 B1). Idempotent; refuses a
|
|
213
|
+
record that already names a DIFFERENT generation or revision (a caller
|
|
214
|
+
must never relabel another protocol's record). The payload is never
|
|
215
|
+
touched: both stamps are transport metadata."""
|
|
154
216
|
if not isinstance(record, dict):
|
|
155
217
|
raise ValueError("journal record must be a dict")
|
|
156
218
|
existing = record.get(WRITER_GENERATION_KEY)
|
|
157
219
|
if existing is not None and existing != PROTOCOL_GENERATION:
|
|
158
220
|
raise ValueError(
|
|
159
221
|
f"record already carries writer generation {existing!r}")
|
|
222
|
+
if REVISION_KEY in record and not has_current_revision(record):
|
|
223
|
+
raise ValueError(
|
|
224
|
+
f"record already carries protocol revision "
|
|
225
|
+
f"{record[REVISION_KEY]!r}")
|
|
160
226
|
out = dict(record)
|
|
161
227
|
out[WRITER_GENERATION_KEY] = PROTOCOL_GENERATION
|
|
228
|
+
out[REVISION_KEY] = PROTOCOL_REVISION
|
|
162
229
|
return out
|
|
163
230
|
|
|
164
231
|
|
|
@@ -167,12 +234,26 @@ def is_stamped(record: dict) -> bool:
|
|
|
167
234
|
and record.get(WRITER_GENERATION_KEY) == PROTOCOL_GENERATION)
|
|
168
235
|
|
|
169
236
|
|
|
237
|
+
def has_current_revision(record: dict) -> bool:
|
|
238
|
+
"""Whether ``record`` carries exactly protocol revision 2 (spec 4.6 B1).
|
|
239
|
+
Only an ``int`` equal to 2: never ``True``, ``"2"``, ``2.0`` or a later
|
|
240
|
+
revision. A future revision is admitted only through an explicit
|
|
241
|
+
compatibility rule added here, never an open-ended ``>= 2``."""
|
|
242
|
+
if not isinstance(record, dict):
|
|
243
|
+
return False
|
|
244
|
+
value = record.get(REVISION_KEY)
|
|
245
|
+
return type(value) is int and value == PROTOCOL_REVISION
|
|
246
|
+
|
|
247
|
+
|
|
170
248
|
def strip_stamp(record: dict) -> dict:
|
|
171
|
-
"""``record`` without its writer-generation
|
|
172
|
-
for semantic event duplicate/conflict
|
|
173
|
-
|
|
249
|
+
"""``record`` without its writer-generation and protocol-revision fields
|
|
250
|
+
— the comparison shape for semantic event duplicate/conflict
|
|
251
|
+
classification."""
|
|
252
|
+
if not isinstance(record, dict) or (
|
|
253
|
+
WRITER_GENERATION_KEY not in record and REVISION_KEY not in record):
|
|
174
254
|
return record
|
|
175
|
-
return {k: v for k, v in record.items()
|
|
255
|
+
return {k: v for k, v in record.items()
|
|
256
|
+
if k not in (WRITER_GENERATION_KEY, REVISION_KEY)}
|
|
176
257
|
|
|
177
258
|
|
|
178
259
|
# --------------------------------------------------------------------------
|
|
@@ -272,6 +353,92 @@ def boundary_from_op(op_record: dict) -> CutoverBoundary:
|
|
|
272
353
|
)
|
|
273
354
|
|
|
274
355
|
|
|
356
|
+
def with_revision(boundary: "CutoverBoundary | None",
|
|
357
|
+
revision: "RevisionBoundary | None"):
|
|
358
|
+
"""``boundary`` carrying ``revision`` (spec 4.6 B3), or None when there is
|
|
359
|
+
no cutover. A revision that names a different cutover is malformed and
|
|
360
|
+
never attached."""
|
|
361
|
+
if boundary is None:
|
|
362
|
+
return None
|
|
363
|
+
if revision is not None and (
|
|
364
|
+
(revision.cutover_segment, revision.cutover_offset)
|
|
365
|
+
!= (boundary.segment, boundary.end_offset)):
|
|
366
|
+
raise ValueError("revision boundary names a different cutover")
|
|
367
|
+
return dataclasses.replace(boundary, revision=revision)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def revision_boundary_payload(boundary: RevisionBoundary) -> dict:
|
|
371
|
+
"""The camelCase payload fields of the revision op (without ``kind``):
|
|
372
|
+
the `boundary_payload` codec, the revision number and the original
|
|
373
|
+
cutover's coordinate."""
|
|
374
|
+
payload = {
|
|
375
|
+
"revision": PROTOCOL_REVISION,
|
|
376
|
+
"cutover": {"segment": boundary.cutover_segment,
|
|
377
|
+
"offset": boundary.cutover_offset},
|
|
378
|
+
"segment": boundary.segment,
|
|
379
|
+
"endOffset": boundary.end_offset,
|
|
380
|
+
"frozenExtents": {k: boundary.frozen_extents[k]
|
|
381
|
+
for k in sorted(boundary.frozen_extents)},
|
|
382
|
+
"legacySpoolBaselineDigest": boundary.legacy_spool_baseline_digest,
|
|
383
|
+
}
|
|
384
|
+
if boundary.legacy_spool_inventory_failure is not None:
|
|
385
|
+
payload["legacySpoolInventoryFailure"] = dict(
|
|
386
|
+
boundary.legacy_spool_inventory_failure)
|
|
387
|
+
if boundary.continued_inventory_episode is not None:
|
|
388
|
+
segment, offset = boundary.continued_inventory_episode
|
|
389
|
+
payload["inventoryEpisode"] = {"segment": segment,
|
|
390
|
+
"offset": offset}
|
|
391
|
+
return payload
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def revision_boundary_from_op(op_record: dict) -> RevisionBoundary:
|
|
395
|
+
"""Decode the revision boundary from an ``account_bound_revision`` op.
|
|
396
|
+
Raises ``ValueError`` on any shape failure, including a revision other
|
|
397
|
+
than exactly 2 and an op that does not itself carry ``wpr == 2``."""
|
|
398
|
+
if not isinstance(op_record, dict) or op_record.get("t") != "op":
|
|
399
|
+
raise ValueError("revision boundary must come from an op record")
|
|
400
|
+
payload = op_record.get("payload")
|
|
401
|
+
if not isinstance(payload, dict) or payload.get("kind") != REVISION_OP_KIND:
|
|
402
|
+
raise ValueError("record is not an account_bound_revision op")
|
|
403
|
+
if type(payload.get("revision")) is not int or (
|
|
404
|
+
payload["revision"] != PROTOCOL_REVISION):
|
|
405
|
+
raise ValueError("revision boundary revision is invalid")
|
|
406
|
+
cutover = payload.get("cutover")
|
|
407
|
+
if (not isinstance(cutover, dict)
|
|
408
|
+
or set(cutover) != {"segment", "offset"}
|
|
409
|
+
or not isinstance(cutover.get("segment"), str)
|
|
410
|
+
or not cutover["segment"]
|
|
411
|
+
or not _nonneg_int(cutover.get("offset"))):
|
|
412
|
+
raise ValueError("revision boundary cutover coordinate is invalid")
|
|
413
|
+
episode = payload.get("inventoryEpisode")
|
|
414
|
+
if episode is not None:
|
|
415
|
+
if (payload.get("legacySpoolInventoryFailure") is None
|
|
416
|
+
or not isinstance(episode, dict)
|
|
417
|
+
or set(episode) != {"segment", "offset"}
|
|
418
|
+
or not isinstance(episode.get("segment"), str)
|
|
419
|
+
or not episode["segment"]
|
|
420
|
+
or not _nonneg_int(episode.get("offset"))):
|
|
421
|
+
raise ValueError("revision boundary inventory episode is invalid")
|
|
422
|
+
episode = (episode["segment"], episode["offset"])
|
|
423
|
+
base = boundary_from_op({
|
|
424
|
+
"t": "op",
|
|
425
|
+
"payload": {**{k: v for k, v in payload.items()
|
|
426
|
+
if k not in ("revision", "cutover",
|
|
427
|
+
"inventoryEpisode")},
|
|
428
|
+
"kind": CUTOVER_OP_KIND},
|
|
429
|
+
})
|
|
430
|
+
return RevisionBoundary(
|
|
431
|
+
segment=base.segment,
|
|
432
|
+
end_offset=base.end_offset,
|
|
433
|
+
frozen_extents=base.frozen_extents,
|
|
434
|
+
legacy_spool_baseline_digest=base.legacy_spool_baseline_digest,
|
|
435
|
+
cutover_segment=cutover["segment"],
|
|
436
|
+
cutover_offset=cutover["offset"],
|
|
437
|
+
legacy_spool_inventory_failure=base.legacy_spool_inventory_failure,
|
|
438
|
+
continued_inventory_episode=episode,
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
275
442
|
# --------------------------------------------------------------------------
|
|
276
443
|
# admission
|
|
277
444
|
# --------------------------------------------------------------------------
|
|
@@ -329,20 +496,36 @@ def record_family(record: dict) -> str:
|
|
|
329
496
|
|
|
330
497
|
|
|
331
498
|
def classify_admission(record: dict, coord: Coordinate,
|
|
332
|
-
boundary: CutoverBoundary | None
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
499
|
+
boundary: CutoverBoundary | None,
|
|
500
|
+
revision_boundary: "RevisionBoundary | None" = None,
|
|
501
|
+
) -> Admission:
|
|
502
|
+
"""The single admission predicate (spec 5.3; #797/#823/#840 spec 4.6 B3).
|
|
503
|
+
|
|
504
|
+
The original rule: admitted when there is no cutover yet, when the
|
|
505
|
+
record lies inside the historical boundary, when it carries the current
|
|
506
|
+
writer generation, or when its family is proven Codex-only; every other
|
|
507
|
+
post-boundary record is excluded.
|
|
508
|
+
|
|
509
|
+
Composed with revision 2: ``revision_boundary`` (default: the revision a
|
|
510
|
+
lookup attached to ``boundary``) excludes, after its own frozen extents,
|
|
511
|
+
a record the original rule admitted only by its writer generation unless
|
|
512
|
+
it also carries ``wpr == 2`` (``revision-unstamped``). The revision never
|
|
513
|
+
re-admits what the original rule excludes, and the Codex-only exemptions
|
|
514
|
+
are unchanged."""
|
|
339
515
|
family, codex_only = _family(record)
|
|
340
516
|
if boundary is None:
|
|
341
517
|
return Admission(True, family, "no-cutover")
|
|
342
518
|
if is_historical(coord, boundary):
|
|
343
519
|
return Admission(True, family, "historical")
|
|
344
520
|
if is_stamped(record):
|
|
345
|
-
|
|
521
|
+
revision = (revision_boundary if revision_boundary is not None
|
|
522
|
+
else boundary.revision)
|
|
523
|
+
if (revision is None or is_historical(coord, revision)
|
|
524
|
+
or has_current_revision(record)):
|
|
525
|
+
return Admission(True, family, "stamped")
|
|
526
|
+
if codex_only:
|
|
527
|
+
return Admission(True, family, "codex-only")
|
|
528
|
+
return Admission(False, family, "revision-unstamped")
|
|
346
529
|
if codex_only:
|
|
347
530
|
return Admission(True, family, "codex-only")
|
|
348
531
|
return Admission(False, family, "excluded")
|
package/bin/_lib_clone_copy.py
CHANGED
|
@@ -12,6 +12,9 @@ for:
|
|
|
12
12
|
existing destination is overwritten in place (same inode), which a clone
|
|
13
13
|
cannot do, so that case is a real copy. Metadata is ``shutil.copystat``'s,
|
|
14
14
|
applied after the clone as after a copy.
|
|
15
|
+
- :func:`try_clone` is the clone alone: True when `dst` was created as a
|
|
16
|
+
clone, False (nothing created) when the platform or filesystem refuses, so
|
|
17
|
+
a caller can choose where its real copy goes (#997 D13).
|
|
15
18
|
- :func:`copytree` is ``shutil.copytree`` with :func:`copy2` as its file
|
|
16
19
|
copier, so symlink policy, ignore lists, dangling-link errors and
|
|
17
20
|
destination collisions are the stdlib's own code.
|
|
@@ -26,11 +29,10 @@ The optional ``counts`` mapping is incremented under ``"clone"`` or
|
|
|
26
29
|
``"copy"`` per file, so a caller or a measurement can see which path ran.
|
|
27
30
|
This module is public and imports nothing outside the standard library.
|
|
28
31
|
|
|
29
|
-
It binds ``fclonefileat`` itself rather than importing the binding
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
from this one instead (spec §6.7).
|
|
32
|
+
It binds ``fclonefileat`` itself rather than importing the binding from the
|
|
33
|
+
retired private staging module (#940 D-2 F17): a private module is absent
|
|
34
|
+
from the public tree that this module and its public callers ship in, and a
|
|
35
|
+
public module may not import it.
|
|
34
36
|
"""
|
|
35
37
|
|
|
36
38
|
from __future__ import annotations
|
|
@@ -85,6 +87,43 @@ def _clone_fd(source_fd: int, destination: str) -> None:
|
|
|
85
87
|
raise OSError(code, os.strerror(code), destination)
|
|
86
88
|
|
|
87
89
|
|
|
90
|
+
def _clone_open_file(fd: int, src, dst, *, follow_symlinks: bool = True) -> bool:
|
|
91
|
+
"""Clone the open regular file `fd` (opened from `src`) to the new path
|
|
92
|
+
`dst` and apply `src`'s metadata. False, with nothing created, when the
|
|
93
|
+
filesystem or the pair of volumes refuses the clone (`_FALLBACK_ERRNOS`);
|
|
94
|
+
any other error propagates."""
|
|
95
|
+
try:
|
|
96
|
+
_clone_fd(fd, dst)
|
|
97
|
+
except OSError as exc:
|
|
98
|
+
if exc.errno not in _FALLBACK_ERRNOS:
|
|
99
|
+
raise
|
|
100
|
+
return False
|
|
101
|
+
shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
|
|
102
|
+
return True
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def try_clone(src, dst) -> bool:
|
|
106
|
+
"""Clone the regular file `src` to the NEW path `dst` (#997 D13).
|
|
107
|
+
|
|
108
|
+
True: `dst` is a clone sharing `src`'s blocks, with ``shutil.copystat``
|
|
109
|
+
metadata applied. False, with `dst` not created: the platform is not
|
|
110
|
+
Darwin (or has no ``fclonefileat``), `src` is not a regular file, or the
|
|
111
|
+
filesystem refused with a fallback errno (an unsupported filesystem, a
|
|
112
|
+
cross-volume destination, an existing `dst`). Any other error, including
|
|
113
|
+
a `src` that cannot be opened, propagates. A caller that needs a copy
|
|
114
|
+
either way makes the real copy itself on False.
|
|
115
|
+
"""
|
|
116
|
+
if _fclonefileat() is None:
|
|
117
|
+
return False
|
|
118
|
+
fd = os.open(src, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
|
|
119
|
+
try:
|
|
120
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
121
|
+
return False
|
|
122
|
+
return _clone_open_file(fd, src, dst)
|
|
123
|
+
finally:
|
|
124
|
+
os.close(fd)
|
|
125
|
+
|
|
126
|
+
|
|
88
127
|
def _count(counts, method: str) -> None:
|
|
89
128
|
if counts is not None:
|
|
90
129
|
counts[method] = counts.get(method, 0) + 1
|
|
@@ -113,15 +152,11 @@ def copy2(src, dst, *, follow_symlinks: bool = True, counts=None):
|
|
|
113
152
|
os.close(fd)
|
|
114
153
|
fd = -1
|
|
115
154
|
return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
except OSError as exc:
|
|
119
|
-
if exc.errno not in _FALLBACK_ERRNOS:
|
|
120
|
-
raise
|
|
155
|
+
if not _clone_open_file(fd, src, dst,
|
|
156
|
+
follow_symlinks=follow_symlinks):
|
|
121
157
|
result = shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
122
158
|
_count(counts, "copy")
|
|
123
159
|
return result
|
|
124
|
-
shutil.copystat(src, dst, follow_symlinks=follow_symlinks)
|
|
125
160
|
_count(counts, "clone")
|
|
126
161
|
return dst
|
|
127
162
|
finally:
|
|
@@ -1079,27 +1079,61 @@ def alerted_at_order_key(value: object) -> str:
|
|
|
1079
1079
|
return _parse_alerted_at(value).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
1080
1080
|
|
|
1081
1081
|
|
|
1082
|
-
def
|
|
1083
|
-
"""
|
|
1084
|
-
|
|
1085
|
-
SQLite's date functions keep only milliseconds, so the fraction is read
|
|
1086
|
-
from the text itself. A UTC offset is a whole number of minutes, so the
|
|
1087
|
-
source fraction is also the UTC instant's fraction.
|
|
1088
|
-
"""
|
|
1082
|
+
def _alerted_at_fraction_digits_sql(column: str) -> str:
|
|
1083
|
+
"""Every digit after the seconds' ``.``, up to the zone designator."""
|
|
1089
1084
|
tail = f"substr({column}, 21)"
|
|
1090
|
-
|
|
1085
|
+
return (
|
|
1091
1086
|
f"substr({tail}, 1, CASE "
|
|
1092
1087
|
f"WHEN instr({tail}, 'Z') > 0 THEN instr({tail}, 'Z') - 1 "
|
|
1093
1088
|
f"WHEN instr({tail}, '+') > 0 THEN instr({tail}, '+') - 1 "
|
|
1094
1089
|
f"WHEN instr({tail}, '-') > 0 THEN instr({tail}, '-') - 1 "
|
|
1095
1090
|
f"ELSE length({tail}) END)"
|
|
1096
1091
|
)
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _alerted_at_fraction_sql(column: str) -> str:
|
|
1095
|
+
"""The six-digit sub-second fraction of an ISO-8601 value, in SQLite.
|
|
1096
|
+
|
|
1097
|
+
SQLite's date functions keep only milliseconds, so the fraction is read
|
|
1098
|
+
from the text itself. A UTC offset is a whole number of minutes, so the
|
|
1099
|
+
source fraction is also the UTC instant's fraction.
|
|
1100
|
+
"""
|
|
1101
|
+
digits = _alerted_at_fraction_digits_sql(column)
|
|
1097
1102
|
return (
|
|
1098
1103
|
f"(CASE WHEN substr({column}, 20, 1) = '.' "
|
|
1099
1104
|
f"THEN substr({digits} || '000000', 1, 6) ELSE '000000' END)"
|
|
1100
1105
|
)
|
|
1101
1106
|
|
|
1102
1107
|
|
|
1108
|
+
def _alerted_at_whole_second_sql(column: str) -> str:
|
|
1109
|
+
"""The value with its sub-second fraction removed, zone kept.
|
|
1110
|
+
|
|
1111
|
+
SQLite before 3.46 rounds a parsed fraction to the nearest millisecond, so
|
|
1112
|
+
``…00.999999`` formats as second ``01`` there and as ``00`` on later
|
|
1113
|
+
versions. The seconds part must come from the whole second alone; the
|
|
1114
|
+
fraction is supplied from the text by :func:`_alerted_at_fraction_sql`.
|
|
1115
|
+
"""
|
|
1116
|
+
digits = _alerted_at_fraction_digits_sql(column)
|
|
1117
|
+
return (
|
|
1118
|
+
f"(CASE WHEN substr({column}, 20, 1) = '.' "
|
|
1119
|
+
f"THEN substr({column}, 1, 19) || "
|
|
1120
|
+
f"substr({column}, 21 + length({digits})) ELSE {column} END)"
|
|
1121
|
+
)
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def _alerted_at_malformed_fraction_sql(column: str) -> str:
|
|
1125
|
+
"""True when the value has a ``.`` fraction that is not all digits.
|
|
1126
|
+
|
|
1127
|
+
:func:`_alerted_at_whole_second_sql` cuts the fraction out before
|
|
1128
|
+
``strftime`` parses the rest, so ``strftime`` alone no longer rejects
|
|
1129
|
+
``…00.abcZ``; this guard keeps such a value ``NULL``, as Python rejects it.
|
|
1130
|
+
"""
|
|
1131
|
+
digits = _alerted_at_fraction_digits_sql(column)
|
|
1132
|
+
return (
|
|
1133
|
+
f"(substr({column}, 20, 1) = '.' AND {digits} GLOB '*[^0-9]*')"
|
|
1134
|
+
)
|
|
1135
|
+
|
|
1136
|
+
|
|
1103
1137
|
def alerted_at_order_key_sql(column: str = "alerted_at") -> str:
|
|
1104
1138
|
"""The SQL twin of :func:`alerted_at_order_key`, for ``ORDER BY … LIMIT``.
|
|
1105
1139
|
|
|
@@ -1111,12 +1145,18 @@ def alerted_at_order_key_sql(column: str = "alerted_at") -> str:
|
|
|
1111
1145
|
every aware spelling this returns the Python helper's string —
|
|
1112
1146
|
``tests/test_556_s3_alert_ordering.py`` pins that agreement. A naive value
|
|
1113
1147
|
is read as UTC here where the Python helper raises; an unparseable value
|
|
1114
|
-
yields ``NULL``, which sorts last under ``DESC``. The
|
|
1115
|
-
|
|
1148
|
+
yields ``NULL``, which sorts last under ``DESC``. The two disagree on
|
|
1149
|
+
spellings Python 3.11+ also accepts (``+05``, ``+0500``, a ``,`` fraction),
|
|
1150
|
+
but every ``alerted_at`` writer stores Python ``isoformat()`` UTC text
|
|
1151
|
+
(``…SS[.ffffff]Z`` or ``…SS[.ffffff]+00:00``), on which they agree. The
|
|
1152
|
+
expression names ``column`` several times, so a bound parameter must be
|
|
1153
|
+
numbered (``?1``).
|
|
1116
1154
|
"""
|
|
1155
|
+
whole = _alerted_at_whole_second_sql(column)
|
|
1117
1156
|
return (
|
|
1118
|
-
f"(
|
|
1119
|
-
f"{
|
|
1157
|
+
f"(CASE WHEN {_alerted_at_malformed_fraction_sql(column)} THEN NULL "
|
|
1158
|
+
f"ELSE strftime('%Y-%m-%dT%H:%M:%S', {whole}) || '.' || "
|
|
1159
|
+
f"{_alerted_at_fraction_sql(column)} || 'Z' END)"
|
|
1120
1160
|
)
|
|
1121
1161
|
|
|
1122
1162
|
|
|
@@ -1130,10 +1170,12 @@ def canonical_alerted_at_sql(column: str = "alerted_at") -> str:
|
|
|
1130
1170
|
:func:`alerted_at_order_key_sql`'s job.
|
|
1131
1171
|
"""
|
|
1132
1172
|
fraction = _alerted_at_fraction_sql(column)
|
|
1173
|
+
whole = _alerted_at_whole_second_sql(column)
|
|
1133
1174
|
return (
|
|
1134
|
-
f"(CASE WHEN {
|
|
1135
|
-
f"
|
|
1136
|
-
f"
|
|
1175
|
+
f"(CASE WHEN {_alerted_at_malformed_fraction_sql(column)} THEN NULL "
|
|
1176
|
+
f"WHEN {fraction} = '000000' "
|
|
1177
|
+
f"THEN strftime('%Y-%m-%dT%H:%M:%SZ', {whole}) "
|
|
1178
|
+
f"ELSE strftime('%Y-%m-%dT%H:%M:%S', {whole}) || '.' || {fraction} "
|
|
1137
1179
|
f"|| 'Z' END)"
|
|
1138
1180
|
)
|
|
1139
1181
|
|